{"item_id":"senseval2.d000.s000.t000","gold_sense_keys":["art%1:09:00::"],"candidates":[{"index":1,"sense_key":"art%1:06:00::","synset_id":"art.n.01"},{"index":2,"sense_key":"art%1:04:00::","synset_id":"art.n.02"},{"index":3,"sense_key":"art%1:09:00::","synset_id":"art.n.03"},{"index":4,"sense_key":"art%1:10:00::","synset_id":"artwork.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"art%1:09:00::","call_ids":["senseval2.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"art%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010264999999999999,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.013926291023381}
{"item_id":"senseval2.d000.s000.t002","gold_sense_keys":["peculiar%5:00:00:characteristic:00","peculiar%5:00:00:specific:00"],"candidates":[{"index":1,"sense_key":"peculiar%5:00:00:strange:00","synset_id":"curious.s.01"},{"index":2,"sense_key":"peculiar%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":3,"sense_key":"peculiar%5:00:00:unusual:00","synset_id":"peculiar.s.03"},{"index":4,"sense_key":"peculiar%5:00:00:characteristic:00","synset_id":"peculiar.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"peculiar%5:00:00:specific:00","call_ids":["senseval2.d000.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"peculiar%5:00:00:specific:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015539999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.701294791011605}
{"item_id":"senseval2.d000.s000.t003","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:10:00::","synset_id":"english.n.01"},{"index":2,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":3,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":4,"sense_key":"english%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022784999999999998,"input_uncached_usd":0.000016649999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.653420958027709}
{"item_id":"senseval2.d000.s000.t004","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d000.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":313,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030365000000000004,"input_uncached_usd":0.00001565,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.3023076669778675}
{"item_id":"senseval2.d000.s000.t005","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:00::","synset_id":"english.a.01"},{"index":2,"sense_key":"english%3:01:01::","synset_id":"english.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":255,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030075,"input_uncached_usd":0.00001275,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2658291250118054}
{"item_id":"senseval2.d000.s000.t006","gold_sense_keys":["peculiarity%1:07:02::","peculiarity%1:09:00::"],"candidates":[{"index":1,"sense_key":"peculiarity%1:09:00::","synset_id":"peculiarity.n.01"},{"index":2,"sense_key":"peculiarity%1:07:02::","synset_id":"peculiarity.n.02"},{"index":3,"sense_key":"peculiarity%1:06:00::","synset_id":"curio.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"peculiarity%1:07:02::","call_ids":["senseval2.d000.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"peculiarity%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":300,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006358000000000001,"input_uncached_usd":0.000014999999999999999,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.496255042031407}
{"item_id":"senseval2.d000.s000.t007","gold_sense_keys":["unintelligible%5:00:00:incomprehensible:00"],"candidates":[{"index":1,"sense_key":"unintelligible%3:00:00::","synset_id":"unintelligible.a.01"},{"index":2,"sense_key":"unintelligible%5:00:00:incomprehensible:00","synset_id":"opaque.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unintelligible%5:00:00:incomprehensible:00","call_ids":["senseval2.d000.s000.t007__v1__a1","senseval2.d000.s000.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unintelligible%5:00:00:incomprehensible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":1824,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00075895,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.0007296,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.184047499962617}
{"item_id":"senseval2.d000.s000.t008","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":6,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":7,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rest%1:24:00::","call_ids":["senseval2.d000.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021059999999999997,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.059818957990501}
{"item_id":"senseval2.d000.s000.t009","gold_sense_keys":["world%1:14:02::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"world%1:14:02::","call_ids":["senseval2.d000.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"world%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049685,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.631669417023659}
{"item_id":"senseval2.d000.s003.t000","gold_sense_keys":["scene%1:09:00::"],"candidates":[{"index":1,"sense_key":"scene%1:15:00::","synset_id":"scene.n.01"},{"index":2,"sense_key":"scene%1:11:00::","synset_id":"scene.n.02"},{"index":3,"sense_key":"scene%1:09:00::","synset_id":"view.n.02"},{"index":4,"sense_key":"scene%1:10:01::","synset_id":"scene.n.04"},{"index":5,"sense_key":"scene%1:26:00::","synset_id":"picture.n.04"},{"index":6,"sense_key":"scene%1:10:00::","synset_id":"scene.n.06"},{"index":7,"sense_key":"scene%1:26:01::","synset_id":"fit.n.01"},{"index":8,"sense_key":"scene%1:06:01::","synset_id":"scene.n.08"},{"index":9,"sense_key":"scene%1:15:01::","synset_id":"setting.n.01"},{"index":10,"sense_key":"scene%1:06:00::","synset_id":"scenery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"scene%1:09:00::","call_ids":["senseval2.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"scene%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007573,"input_uncached_usd":0.000034099999999999995,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.7530902090366}
{"item_id":"senseval2.d000.s003.t001","gold_sense_keys":["evoke%2:32:01::"],"candidates":[{"index":1,"sense_key":"evoke%2:37:00::","synset_id":"arouse.v.01"},{"index":2,"sense_key":"evoke%2:36:00::","synset_id":"provoke.v.02"},{"index":3,"sense_key":"evoke%2:36:01::","synset_id":"educe.v.01"},{"index":4,"sense_key":"evoke%2:36:02::","synset_id":"raise.v.07"},{"index":5,"sense_key":"evoke%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"evoke%2:32:01::","call_ids":["senseval2.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"evoke%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006476,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.329838916019071}
{"item_id":"senseval2.d000.s003.t004","gold_sense_keys":["lovely%5:00:00:beautiful:00"],"candidates":[{"index":1,"sense_key":"lovely%5:00:00:beautiful:00","synset_id":"lovely.s.01"},{"index":2,"sense_key":"lovely%5:00:00:lovable:00","synset_id":"adorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lovely%5:00:00:beautiful:00","call_ids":["senseval2.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lovely%5:00:00:beautiful:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020245,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.786675790965091}
{"item_id":"senseval2.d000.s003.t005","gold_sense_keys":["ancient%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"ancient%5:00:00:past:00","synset_id":"ancient.s.01"},{"index":2,"sense_key":"ancient%5:00:00:old:02","synset_id":"ancient.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ancient%5:00:00:old:02","call_ids":["senseval2.d000.s003.t005__v1__a1","senseval2.d000.s003.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ancient%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":2272,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009470499999999998,"input_uncached_usd":0.000038249999999999995,"input_cached_usd":0.0,"output_usd":0.0009088,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.580564834002871}
{"item_id":"senseval2.d000.s003.t006","gold_sense_keys":["stone%1:06:00::"],"candidates":[{"index":1,"sense_key":"stone%1:17:00::","synset_id":"rock.n.01"},{"index":2,"sense_key":"stone%1:06:00::","synset_id":"stone.n.02"},{"index":3,"sense_key":"stone%1:27:00::","synset_id":"rock.n.02"},{"index":4,"sense_key":"stone%1:27:01::","synset_id":"gem.n.02"},{"index":5,"sense_key":"stone%1:23:00::","synset_id":"stone.n.05"},{"index":6,"sense_key":"stone%1:20:00::","synset_id":"stone.n.06"},{"index":7,"sense_key":"stone%1:18:05::","synset_id":"stone.n.07"},{"index":8,"sense_key":"stone%1:18:04::","synset_id":"stone.n.08"},{"index":9,"sense_key":"stone%1:18:03::","synset_id":"stone.n.09"},{"index":10,"sense_key":"stone%1:18:02::","synset_id":"stone.n.10"},{"index":11,"sense_key":"stone%1:18:01::","synset_id":"stone.n.11"},{"index":12,"sense_key":"stone%1:18:00::","synset_id":"stone.n.12"},{"index":13,"sense_key":"stone%1:07:00::","synset_id":"stone.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stone%1:06:00::","call_ids":["senseval2.d000.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stone%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030665,"input_uncached_usd":0.00004425,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.825618957984261}
{"item_id":"senseval2.d000.s003.t007","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041125,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.050186084001325}
{"item_id":"senseval2.d000.s003.t008","gold_sense_keys":["stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stand%2:42:03::","call_ids":["senseval2.d000.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stand%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011934999999999999,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.790455624985043}
{"item_id":"senseval2.d000.s003.t009","gold_sense_keys":["field%1:15:00::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"field%1:17:00::","call_ids":["senseval2.d000.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"field%1:17:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1038,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004423,"input_uncached_usd":0.000051899999999999994,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.874053416016977}
{"item_id":"senseval2.d000.s003.t010","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s003.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031695,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.2779033340048045}
{"item_id":"senseval2.d000.s003.t011","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell%1:11:00::","call_ids":["senseval2.d000.s003.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011784999999999999,"input_uncached_usd":0.000034649999999999995,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.590757499972824}
{"item_id":"senseval2.d000.s003.t012","gold_sense_keys":["cascade%2:38:00::"],"candidates":[{"index":1,"sense_key":"cascade%2:38:00::","synset_id":"cascade.v.01"},{"index":2,"sense_key":"cascade%2:35:00::","synset_id":"cascade.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cascade%2:38:00::","call_ids":["senseval2.d000.s003.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cascade%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035604999999999995,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.847693749994505}
{"item_id":"senseval2.d000.s003.t013","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s003.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020504999999999997,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.186864790972322}
{"item_id":"senseval2.d000.s003.t014","gold_sense_keys":["call%2:32:05::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"call%2:41:04::","call_ids":["senseval2.d000.s003.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"call%2:41:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1443,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00069295,"input_uncached_usd":0.00007214999999999999,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.498656167008448}
{"item_id":"senseval2.d000.s003.t015","gold_sense_keys":["faithful%1:14:00::"],"candidates":[{"index":1,"sense_key":"faithful%1:14:01::","synset_id":"faithful.n.01"},{"index":2,"sense_key":"faithful%1:14:00::","synset_id":"congregation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"faithful%1:14:00::","call_ids":["senseval2.d000.s003.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"faithful%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":331,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006373500000000001,"input_uncached_usd":0.00001655,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.77682020800421}
{"item_id":"senseval2.d000.s003.t016","gold_sense_keys":["evensong%1:10:00::"],"candidates":[{"index":1,"sense_key":"evensong%1:28:00::","synset_id":"vespers.n.01"},{"index":2,"sense_key":"evensong%1:10:00::","synset_id":"evening_prayer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evensong%1:10:00::","call_ids":["senseval2.d000.s003.t016__v1__a1","senseval2.d000.s003.t016__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evensong%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":2912,"reasoning_output_tokens":2880},"cost":{"currency":"USD","total_usd":0.0012028499999999999,"input_uncached_usd":0.00003805,"input_cached_usd":0.0,"output_usd":0.0011648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":23.427610832965}
{"item_id":"senseval2.d000.s004.t001","gold_sense_keys":["stop%2:38:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stop%2:38:00::","call_ids":["senseval2.d000.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stop%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045654999999999996,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.024485624977387}
{"item_id":"senseval2.d000.s004.t003","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023344999999999998,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.527800916985143}
{"item_id":"senseval2.d000.s004.t004","gold_sense_keys":["door%1:06:00::","door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval2.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031485,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.16849925002316}
{"item_id":"senseval2.d000.s004.t005","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d000.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003408,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.367666665988509}
{"item_id":"senseval2.d000.s004.t006","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003615499999999999,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.4725136660272256}
{"item_id":"senseval2.d000.s004.t007","gold_sense_keys":["always%4:02:00::"],"candidates":[{"index":1,"sense_key":"always%4:02:00::","synset_id":"always.r.01"},{"index":2,"sense_key":"always%4:02:04::","synset_id":"constantly.r.01"},{"index":3,"sense_key":"always%4:02:03::","synset_id":"constantly.r.02"},{"index":4,"sense_key":"always%4:02:02::","synset_id":"always.r.04"},{"index":5,"sense_key":"always%4:02:01::","synset_id":"always.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"always%4:02:04::","call_ids":["senseval2.d000.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"always%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059555,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.664448333031032}
{"item_id":"senseval2.d000.s005.t000","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013154999999999997,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8413373749936}
{"item_id":"senseval2.d000.s005.t001","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d000.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005105999999999999,"input_uncached_usd":0.0000434,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.943741208990104}
{"item_id":"senseval2.d000.s005.t002","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039125,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.750628375040833}
{"item_id":"senseval2.d000.s005.t003","gold_sense_keys":["pull%2:35:04::"],"candidates":[{"index":1,"sense_key":"pull%2:35:00::","synset_id":"pull.v.01"},{"index":2,"sense_key":"pull%2:35:02::","synset_id":"attract.v.01"},{"index":3,"sense_key":"pull%2:38:01::","synset_id":"pull.v.03"},{"index":4,"sense_key":"pull%2:35:04::","synset_id":"pull.v.04"},{"index":5,"sense_key":"pull%2:41:00::","synset_id":"perpetrate.v.01"},{"index":6,"sense_key":"pull%2:38:03::","synset_id":"draw.v.05"},{"index":7,"sense_key":"pull%2:38:00::","synset_id":"pull.v.07"},{"index":8,"sense_key":"pull%2:29:00::","synset_id":"pull.v.08"},{"index":9,"sense_key":"pull%2:38:13::","synset_id":"pull.v.09"},{"index":10,"sense_key":"pull%2:38:02::","synset_id":"pull.v.10"},{"index":11,"sense_key":"pull%2:38:04::","synset_id":"pull.v.11"},{"index":12,"sense_key":"pull%2:35:01::","synset_id":"rend.v.01"},{"index":13,"sense_key":"pull%2:35:03::","synset_id":"pull.v.13"},{"index":14,"sense_key":"pull%2:35:05::","synset_id":"pluck.v.05"},{"index":15,"sense_key":"pull%2:35:10::","synset_id":"extract.v.01"},{"index":16,"sense_key":"pull%2:33:13::","synset_id":"pull.v.16"},{"index":17,"sense_key":"pull%2:30:11::","synset_id":"pull.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pull%2:35:04::","call_ids":["senseval2.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pull%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1086,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002911,"input_uncached_usd":0.0000543,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.71233570802724}
{"item_id":"senseval2.d000.s005.t005","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043595,"input_uncached_usd":0.00001995,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.9145994579885155}
{"item_id":"senseval2.d000.s005.t007","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d000.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028704999999999996,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9541619590017945}
{"item_id":"senseval2.d000.s005.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s005.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022354999999999999,"input_uncached_usd":0.00003795,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.121227291994728}
{"item_id":"senseval2.d000.s005.t009","gold_sense_keys":["first%4:02:02::"],"candidates":[{"index":1,"sense_key":"first%4:02:00::","synset_id":"first.r.01"},{"index":2,"sense_key":"first%4:02:02::","synset_id":"first.r.02"},{"index":3,"sense_key":"first%4:02:03::","synset_id":"first.r.03"},{"index":4,"sense_key":"first%4:02:01::","synset_id":"foremost.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%4:02:02::","call_ids":["senseval2.d000.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044039999999999997,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.924912250018679}
{"item_id":"senseval2.d000.s005.t010","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":2,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":3,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":4,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":5,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":6,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":7,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sound%2:39:03::","call_ids":["senseval2.d000.s005.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sound%2:39:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005976000000000001,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.197089624998625}
{"item_id":"senseval2.d000.s005.t011","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["senseval2.d000.s005.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033785,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.20450237498153}
{"item_id":"senseval2.d000.s006.t001","gold_sense_keys":["discordant%3:00:00::"],"candidates":[{"index":1,"sense_key":"discordant%3:00:00::","synset_id":"discordant.a.01"},{"index":2,"sense_key":"discordant%5:00:00:inharmonious:00","synset_id":"discordant.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discordant%5:00:00:inharmonious:00","call_ids":["senseval2.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discordant%5:00:00:inharmonious:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033444999999999997,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.486938832968008}
{"item_id":"senseval2.d000.s006.t003","gold_sense_keys":["note%1:07:00::"],"candidates":[{"index":1,"sense_key":"note%1:10:00::","synset_id":"note.n.01"},{"index":2,"sense_key":"note%1:10:03::","synset_id":"note.n.02"},{"index":3,"sense_key":"note%1:10:04::","synset_id":"note.n.03"},{"index":4,"sense_key":"note%1:10:05::","synset_id":"note.n.04"},{"index":5,"sense_key":"note%1:07:00::","synset_id":"note.n.05"},{"index":6,"sense_key":"note%1:21:01::","synset_id":"bill.n.03"},{"index":7,"sense_key":"note%1:10:02::","synset_id":"note.n.07"},{"index":8,"sense_key":"note%1:26:00::","synset_id":"eminence.n.01"},{"index":9,"sense_key":"note%1:21:00::","synset_id":"note.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"note%1:07:00::","call_ids":["senseval2.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"note%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004029,"input_uncached_usd":0.0000377,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0939330420224}
{"item_id":"senseval2.d000.s006.t005","gold_sense_keys":["enjoy%2:34:00::","enjoy%2:37:00::","enjoy%2:37:01::"],"candidates":[{"index":1,"sense_key":"enjoy%2:37:00::","synset_id":"enjoy.v.01"},{"index":2,"sense_key":"enjoy%2:34:01::","synset_id":"enjoy.v.02"},{"index":3,"sense_key":"enjoy%2:37:01::","synset_id":"love.v.02"},{"index":4,"sense_key":"enjoy%2:39:00::","synset_id":"enjoy.v.04"},{"index":5,"sense_key":"enjoy%2:34:00::","synset_id":"delight.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"enjoy%2:37:00::","call_ids":["senseval2.d000.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"enjoy%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033985,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.3975527079892345}
{"item_id":"senseval2.d000.s006.t007","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell%1:11:00::","call_ids":["senseval2.d000.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005313500000000001,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.3594269999885}
{"item_id":"senseval2.d000.s006.t008","gold_sense_keys":["cool%3:00:01::"],"candidates":[{"index":1,"sense_key":"cool%3:00:01::","synset_id":"cool.a.01"},{"index":2,"sense_key":"cool%5:00:00:composed:00","synset_id":"cool.s.01"},{"index":3,"sense_key":"cool%3:00:03::","synset_id":"cool.a.03"},{"index":4,"sense_key":"cool%3:00:02::","synset_id":"cool.a.04"},{"index":5,"sense_key":"cool%5:00:00:unqualified:02","synset_id":"cool.s.02"},{"index":6,"sense_key":"cool%5:00:00:fashionable:00","synset_id":"cool.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cool%3:00:01::","call_ids":["senseval2.d000.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cool%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013915,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.812167000025511}
{"item_id":"senseval2.d000.s006.t010","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval2.d000.s006.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020935,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.530130958009977}
{"item_id":"senseval2.d000.s007.t000","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d000.s007.t000__v1__a1","senseval2.d000.s007.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":985,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00059965,"input_uncached_usd":0.00004925,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.72888954199152}
{"item_id":"senseval2.d000.s007.t001","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d000.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002116,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.499351334001403}
{"item_id":"senseval2.d000.s007.t002","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015935,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.731603499967605}
{"item_id":"senseval2.d000.s007.t004","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":2,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":3,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":4,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":5,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":6,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":7,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":8,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":9,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":10,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":11,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":12,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":13,"sense_key":"set%1:06:01::","synset_id":"set.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%1:14:00::","call_ids":["senseval2.d000.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":904,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003332,"input_uncached_usd":0.0000452,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.649992291990202}
{"item_id":"senseval2.d000.s007.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048024999999999994,"input_uncached_usd":0.00003865,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.785690542019438}
{"item_id":"senseval2.d000.s007.t006","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d000.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013189999999999998,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9417725420207717}
{"item_id":"senseval2.d000.s007.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s007.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015854999999999998,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.302323832991533}
{"item_id":"senseval2.d000.s007.t011","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s007.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006678,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.146024833025876}
{"item_id":"senseval2.d000.s007.t012","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"},{"index":2,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":3,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":4,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morning%1:28:00::","call_ids":["senseval2.d000.s007.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023694999999999999,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.961456165998243}
{"item_id":"senseval2.d000.s007.t013","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval2.d000.s007.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036314999999999996,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.898920290987007}
{"item_id":"senseval2.d000.s007.t014","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s007.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":925,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00012944999999999998,"input_uncached_usd":0.00004625,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9965428749565035}
{"item_id":"senseval2.d000.s008.t000","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"now%4:02:07::","call_ids":["senseval2.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"now%4:02:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034629999999999996,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.619884249987081}
{"item_id":"senseval2.d000.s008.t001","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003193,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.476212417008355}
{"item_id":"senseval2.d000.s008.t002","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval2.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013204999999999998,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.1809455420007}
{"item_id":"senseval2.d000.s008.t003","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015854999999999998,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.551492209022399}
{"item_id":"senseval2.d000.s008.t004","gold_sense_keys":["remain%2:42:05::"],"candidates":[{"index":1,"sense_key":"remain%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"remain%2:42:03::","synset_id":"stay.v.04"},{"index":3,"sense_key":"remain%2:42:05::","synset_id":"remain.v.03"},{"index":4,"sense_key":"remain%2:42:01::","synset_id":"persist.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"remain%2:42:05::","call_ids":["senseval2.d000.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"remain%2:42:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018795,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0076792080071755}
{"item_id":"senseval2.d000.s009.t000","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"here%4:02:01::","call_ids":["senseval2.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"here%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006207000000000001,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.690243916003965}
{"item_id":"senseval2.d000.s009.t001","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s009.t001__v1__a1","senseval2.d000.s009.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006473499999999999,"input_uncached_usd":0.00004575,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.978467290988192}
{"item_id":"senseval2.d000.s009.t002","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval2.d000.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019534999999999998,"input_uncached_usd":0.00003535,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.611028584011365}
{"item_id":"senseval2.d000.s010.t001","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.000338,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.378618375048973}
{"item_id":"senseval2.d000.s010.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.000185,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9046091249911115}
{"item_id":"senseval2.d000.s010.t003","gold_sense_keys":["include%2:42:00::"],"candidates":[{"index":1,"sense_key":"include%2:42:00::","synset_id":"include.v.01"},{"index":2,"sense_key":"include%2:31:00::","synset_id":"include.v.02"},{"index":3,"sense_key":"include%2:30:00::","synset_id":"include.v.03"},{"index":4,"sense_key":"include%2:41:03::","synset_id":"admit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"include%2:30:00::","call_ids":["senseval2.d000.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"include%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013619999999999998,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.525838291971013}
{"item_id":"senseval2.d000.s010.t006","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["senseval2.d000.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061695,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.774809624999762}
{"item_id":"senseval2.d000.s010.t007","gold_sense_keys":["drive%2:38:00::"],"candidates":[{"index":1,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":2,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":3,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":4,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":5,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":6,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":7,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":8,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":9,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":10,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":11,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":12,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":13,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":14,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":15,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":16,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":17,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":18,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":19,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":20,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":21,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":22,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drive%2:38:00::","call_ids":["senseval2.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drive%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1223,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00037475,"input_uncached_usd":0.00006115,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.854479957954027}
{"item_id":"senseval2.d000.s010.t008","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004126,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.436417041986715}
{"item_id":"senseval2.d000.s010.t009","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039049999999999995,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.582509832980577}
{"item_id":"senseval2.d000.s010.t010","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033929999999999995,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.85768883296987}
{"item_id":"senseval2.d000.s010.t011","gold_sense_keys":["effort%1:04:00::"],"candidates":[{"index":1,"sense_key":"effort%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"effort%1:04:01::","synset_id":"effort.n.02"},{"index":3,"sense_key":"effort%1:04:02::","synset_id":"feat.n.01"},{"index":4,"sense_key":"effort%1:04:03::","synset_id":"campaign.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"effort%1:04:01::","call_ids":["senseval2.d000.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"effort%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":335,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016295,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.000134,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.807764625002164}
{"item_id":"senseval2.d000.s010.t012","gold_sense_keys":["keep%2:42:00::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"keep%2:42:00::","call_ids":["senseval2.d000.s010.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"keep%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1293,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00048065,"input_uncached_usd":0.00006465,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.216757040994707}
{"item_id":"senseval2.d000.s010.t013","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s010.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006347000000000001,"input_uncached_usd":0.0000395,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.977435874985531}
{"item_id":"senseval2.d000.s010.t014","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":2,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":3,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":4,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":5,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":6,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":7,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"sound%2:39:01::","call_ids":["senseval2.d000.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"sound%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044554999999999996,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.081914958020207}
{"item_id":"senseval2.d000.s011.t000","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d000.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036574999999999997,"input_uncached_usd":0.00002615,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.325002041005064}
{"item_id":"senseval2.d000.s011.t001","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020684999999999999,"input_uncached_usd":0.00004685,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.311925542016979}
{"item_id":"senseval2.d000.s011.t002","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033765,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.238094542000908}
{"item_id":"senseval2.d000.s011.t003","gold_sense_keys":["scrape%2:35:03::"],"candidates":[{"index":1,"sense_key":"scrape%2:35:00::","synset_id":"scrape.v.01"},{"index":2,"sense_key":"scrape%2:36:00::","synset_id":"scrape.v.02"},{"index":3,"sense_key":"scrape%2:35:02::","synset_id":"scratch.v.02"},{"index":4,"sense_key":"scrape%2:38:00::","synset_id":"scrape.v.04"},{"index":5,"sense_key":"scrape%2:35:03::","synset_id":"scrape.v.05"},{"index":6,"sense_key":"scrape%2:35:01::","synset_id":"skin.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"scrape%2:35:03::","call_ids":["senseval2.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"scrape%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034355,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.596937750000507}
{"item_id":"senseval2.d000.s011.t004","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030429999999999997,"input_uncached_usd":0.000041899999999999995,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.628245584026445}
{"item_id":"senseval2.d000.s011.t006","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["senseval2.d000.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044209999999999996,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.900018249987625}
{"item_id":"senseval2.d000.s012.t000","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval2.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030189999999999997,"input_uncached_usd":0.0000395,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.254063915985171}
{"item_id":"senseval2.d000.s012.t001","gold_sense_keys":["train%2:31:01::","train%2:32:00::","train%2:41:00::"],"candidates":[{"index":1,"sense_key":"train%2:31:01::","synset_id":"train.v.01"},{"index":2,"sense_key":"train%2:31:00::","synset_id":"train.v.02"},{"index":3,"sense_key":"train%2:41:02::","synset_id":"discipline.v.01"},{"index":4,"sense_key":"train%2:41:00::","synset_id":"prepare.v.05"},{"index":5,"sense_key":"train%2:41:01::","synset_id":"educate.v.03"},{"index":6,"sense_key":"train%2:33:00::","synset_id":"aim.v.01"},{"index":7,"sense_key":"train%2:32:00::","synset_id":"coach.v.01"},{"index":8,"sense_key":"train%2:29:00::","synset_id":"train.v.08"},{"index":9,"sense_key":"train%2:41:03::","synset_id":"train.v.09"},{"index":10,"sense_key":"train%2:38:00::","synset_id":"train.v.10"},{"index":11,"sense_key":"train%2:35:04::","synset_id":"trail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"train%2:32:00::","call_ids":["senseval2.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"train%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007421000000000001,"input_uncached_usd":0.0000445,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.575357040972449}
{"item_id":"senseval2.d000.s012.t003","gold_sense_keys":["disco%1:06:00::"],"candidates":[{"index":1,"sense_key":"disco%1:10:00::","synset_id":"disco.n.01"},{"index":2,"sense_key":"disco%1:06:00::","synset_id":"disco.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"disco%1:06:00::","call_ids":["senseval2.d000.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"disco%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046359999999999994,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.424681290984154}
{"item_id":"senseval2.d000.s012.t004","gold_sense_keys":["dance%1:11:00::","dance%1:14:00::"],"candidates":[{"index":1,"sense_key":"dance%1:10:00::","synset_id":"dance.n.01"},{"index":2,"sense_key":"dance%1:14:00::","synset_id":"dance.n.02"},{"index":3,"sense_key":"dance%1:04:00::","synset_id":"dancing.n.01"},{"index":4,"sense_key":"dance%1:11:00::","synset_id":"dance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"dance%1:11:00::","call_ids":["senseval2.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"dance%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004137,"input_uncached_usd":0.0000233,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.281882417039014}
{"item_id":"senseval2.d000.s012.t005","gold_sense_keys":["just%4:02:03::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"just%4:02:00::","call_ids":["senseval2.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"just%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039694999999999997,"input_uncached_usd":0.000032149999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.906089833006263}
{"item_id":"senseval2.d000.s012.t006","gold_sense_keys":["drift%2:38:04::"],"candidates":[{"index":1,"sense_key":"drift%2:38:02::","synset_id":"float.v.01"},{"index":2,"sense_key":"drift%2:38:04::","synset_id":"stray.v.02"},{"index":3,"sense_key":"drift%2:38:01::","synset_id":"roll.v.12"},{"index":4,"sense_key":"drift%2:42:03::","synset_id":"drift.v.04"},{"index":5,"sense_key":"drift%2:42:00::","synset_id":"freewheel.v.01"},{"index":6,"sense_key":"drift%2:38:05::","synset_id":"drift.v.06"},{"index":7,"sense_key":"drift%2:38:06::","synset_id":"drift.v.07"},{"index":8,"sense_key":"drift%2:35:03::","synset_id":"drift.v.08"},{"index":9,"sense_key":"drift%2:30:10::","synset_id":"drift.v.09"},{"index":10,"sense_key":"drift%2:30:00::","synset_id":"drift.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drift%2:38:04::","call_ids":["senseval2.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drift%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004562,"input_uncached_usd":0.0000402,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.943979625008069}
{"item_id":"senseval2.d000.s012.t007","gold_sense_keys":["away%4:02:00::"],"candidates":[{"index":1,"sense_key":"away%4:02:00::","synset_id":"away.r.01"},{"index":2,"sense_key":"away%4:02:01::","synset_id":"away.r.02"},{"index":3,"sense_key":"away%4:02:03::","synset_id":"aside.r.02"},{"index":4,"sense_key":"away%4:02:04::","synset_id":"away.r.04"},{"index":5,"sense_key":"away%4:02:08::","synset_id":"off.r.02"},{"index":6,"sense_key":"away%4:02:05::","synset_id":"away.r.06"},{"index":7,"sense_key":"away%4:02:09::","synset_id":"away.r.07"},{"index":8,"sense_key":"away%4:02:10::","synset_id":"away.r.08"},{"index":9,"sense_key":"away%4:02:07::","synset_id":"away.r.09"},{"index":10,"sense_key":"away%4:02:06::","synset_id":"away.r.10"},{"index":11,"sense_key":"away%4:02:02::","synset_id":"aside.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"away%4:02:00::","call_ids":["senseval2.d000.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"away%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038109999999999994,"input_uncached_usd":0.000041899999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.203741832985543}
{"item_id":"senseval2.d000.s013.t000","gold_sense_keys":["worry%2:37:00::"],"candidates":[{"index":1,"sense_key":"worry%2:37:00::","synset_id":"worry.v.01"},{"index":2,"sense_key":"worry%2:37:12::","synset_id":"worry.v.02"},{"index":3,"sense_key":"worry%2:37:01::","synset_id":"worry.v.03"},{"index":4,"sense_key":"worry%2:42:00::","synset_id":"concern.v.02"},{"index":5,"sense_key":"worry%2:35:00::","synset_id":"worry.v.05"},{"index":6,"sense_key":"worry%2:35:01::","synset_id":"worry.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worry%2:37:00::","call_ids":["senseval2.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worry%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002896,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.54315670800861}
{"item_id":"senseval2.d000.s013.t003","gold_sense_keys":["youth%1:07:00::","youth%1:14:00::","youth%1:26:00::","youth%1:28:00::"],"candidates":[{"index":1,"sense_key":"youth%1:18:00::","synset_id":"young_person.n.01"},{"index":2,"sense_key":"youth%1:14:00::","synset_id":"young.n.09"},{"index":3,"sense_key":"youth%1:28:00::","synset_id":"youth.n.03"},{"index":4,"sense_key":"youth%1:26:00::","synset_id":"youth.n.04"},{"index":5,"sense_key":"youth%1:28:01::","synset_id":"youth.n.05"},{"index":6,"sense_key":"youth%1:07:00::","synset_id":"youth.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"youth%1:26:00::","call_ids":["senseval2.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"youth%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039109999999999997,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.483773417014163}
{"item_id":"senseval2.d000.s013.t005","gold_sense_keys":["rank%1:14:02::"],"candidates":[{"index":1,"sense_key":"rank%1:14:00::","synset_id":"rank.n.01"},{"index":2,"sense_key":"rank%1:26:00::","synset_id":"rank.n.02"},{"index":3,"sense_key":"rank%1:14:01::","synset_id":"rank_and_file.n.01"},{"index":4,"sense_key":"rank%1:26:01::","synset_id":"social_station.n.01"},{"index":5,"sense_key":"rank%1:14:02::","synset_id":"membership.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"rank%1:14:02::","call_ids":["senseval2.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"rank%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036759999999999994,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.246565667039249}
{"item_id":"senseval2.d000.s013.t006","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013089999999999998,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6612830000231043}
{"item_id":"senseval2.d000.s013.t007","gold_sense_keys":["keep%2:42:00::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval2.d000.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"keep%2:42:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1256,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004276,"input_uncached_usd":0.0000628,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.547373834007885}
{"item_id":"senseval2.d000.s013.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005564,"input_uncached_usd":0.0000376,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.774924541998189}
{"item_id":"senseval2.d000.s014.t000","gold_sense_keys":["history%1:28:00::"],"candidates":[{"index":1,"sense_key":"history%1:28:00::","synset_id":"history.n.01"},{"index":2,"sense_key":"history%1:10:00::","synset_id":"history.n.02"},{"index":3,"sense_key":"history%1:09:00::","synset_id":"history.n.03"},{"index":4,"sense_key":"history%1:28:02::","synset_id":"history.n.04"},{"index":5,"sense_key":"history%1:09:01::","synset_id":"history.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"history%1:28:02::","call_ids":["senseval2.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"history%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039255,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.313813041022513}
{"item_id":"senseval2.d000.s014.t001","gold_sense_keys":["side%1:10:00::","side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"side%1:10:00::","call_ids":["senseval2.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"side%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003816499999999999,"input_uncached_usd":0.000042449999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.481512458994985}
{"item_id":"senseval2.d000.s015.t001","gold_sense_keys":["survey%1:04:02::"],"candidates":[{"index":1,"sense_key":"survey%1:04:02::","synset_id":"survey.n.01"},{"index":2,"sense_key":"survey%1:10:00::","synset_id":"sketch.n.03"},{"index":3,"sense_key":"survey%1:04:01::","synset_id":"view.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survey%1:04:02::","call_ids":["senseval2.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survey%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046424999999999993,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.44853716599755}
{"item_id":"senseval2.d000.s015.t002","gold_sense_keys":["take%2:41:04::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"take%2:41:04::","call_ids":["senseval2.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"take%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2211,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00027055,"input_uncached_usd":0.00011054999999999999,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4498010000097565}
{"item_id":"senseval2.d000.s015.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002384,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.721515292010736}
{"item_id":"senseval2.d000.s015.t007","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003386,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.924640583980363}
{"item_id":"senseval2.d000.s015.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022439999999999998,"input_uncached_usd":0.0000388,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.248791749996599}
{"item_id":"senseval2.d000.s015.t009","gold_sense_keys":["no%4:02:01::"],"candidates":[{"index":1,"sense_key":"no%4:02:02::","synset_id":"no.r.01"},{"index":2,"sense_key":"no%4:02:01::","synset_id":"no.r.02"},{"index":3,"sense_key":"no%4:02:00::","synset_id":"no.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"no%4:02:02::","call_ids":["senseval2.d000.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"no%4:02:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":2832,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.00115595,"input_uncached_usd":0.00002315,"input_cached_usd":0.0,"output_usd":0.0011328,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":19.977340332989115}
{"item_id":"senseval2.d000.s015.t011","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s015.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001659,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.285722333006561}
{"item_id":"senseval2.d000.s015.t012","gold_sense_keys":["sunday%1:28:00::"],"candidates":[{"index":1,"sense_key":"sunday%1:28:00::","synset_id":"sunday.n.01"},{"index":2,"sense_key":"sunday%1:18:00::","synset_id":"sunday.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sunday%1:28:00::","call_ids":["senseval2.d000.s015.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sunday%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030955,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.4507117079920135}
{"item_id":"senseval2.d000.s015.t013","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s015.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026825,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.337888207985088}
{"item_id":"senseval2.d000.s016.t000","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":2,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":3,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":4,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":5,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":6,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":7,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":8,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":9,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":10,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":11,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":12,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"easy%3:00:01::","call_ids":["senseval2.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050985,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.093611584044993}
{"item_id":"senseval2.d000.s016.t001","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval2.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1472,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000336,"input_uncached_usd":0.0000736,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.8601716659613885}
{"item_id":"senseval2.d000.s016.t002","gold_sense_keys":["ancient%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"ancient%5:00:00:past:00","synset_id":"ancient.s.01"},{"index":2,"sense_key":"ancient%5:00:00:old:02","synset_id":"ancient.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ancient%5:00:00:old:02","call_ids":["senseval2.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ancient%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003859,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0677200829959475}
{"item_id":"senseval2.d000.s016.t003","gold_sense_keys":["art%1:09:00::"],"candidates":[{"index":1,"sense_key":"art%1:06:00::","synset_id":"art.n.01"},{"index":2,"sense_key":"art%1:04:00::","synset_id":"art.n.02"},{"index":3,"sense_key":"art%1:09:00::","synset_id":"art.n.03"},{"index":4,"sense_key":"art%1:10:00::","synset_id":"artwork.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"art%1:09:00::","call_ids":["senseval2.d000.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"art%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018634999999999998,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.100241084001027}
{"item_id":"senseval2.d000.s017.t000","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d000.s017.t000__v1__a1","senseval2.d000.s017.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006964499999999999,"input_uncached_usd":0.000043650000000000004,"input_cached_usd":0.0,"output_usd":0.0006527999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.049491331970785}
{"item_id":"senseval2.d000.s017.t002","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d000.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034314999999999997,"input_uncached_usd":0.000029149999999999997,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.422672707994934}
{"item_id":"senseval2.d000.s017.t003","gold_sense_keys":["play%2:36:00::","play%2:36:01::","play%2:36:12::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"play%2:36:01::","call_ids":["senseval2.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"play%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1871,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00048395,"input_uncached_usd":0.00009355,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.356907666020561}
{"item_id":"senseval2.d000.s017.t004","gold_sense_keys":["tune%1:10:00::"],"candidates":[{"index":1,"sense_key":"tune%1:10:00::","synset_id":"tune.n.01"},{"index":2,"sense_key":"tune%1:07:00::","synset_id":"tune.n.02"},{"index":3,"sense_key":"tune%1:04:00::","synset_id":"tune.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tune%1:10:00::","call_ids":["senseval2.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tune%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000311,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.68083554104669}
{"item_id":"senseval2.d000.s017.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032605,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.453238999994937}
{"item_id":"senseval2.d000.s017.t008","gold_sense_keys":["europe%1:14:01::","europe%1:17:00::"],"candidates":[{"index":1,"sense_key":"europe%1:17:00::","synset_id":"europe.n.01"},{"index":2,"sense_key":"europe%1:14:00::","synset_id":"european_union.n.01"},{"index":3,"sense_key":"europe%1:14:01::","synset_id":"europe.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"europe%1:17:00::","call_ids":["senseval2.d000.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"europe%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021004999999999998,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.020502833998762}
{"item_id":"senseval2.d000.s017.t009","gold_sense_keys":["consider%2:31:00::"],"candidates":[{"index":1,"sense_key":"consider%2:31:00::","synset_id":"see.v.05"},{"index":2,"sense_key":"consider%2:39:00::","synset_id":"study.v.03"},{"index":3,"sense_key":"consider%2:31:01::","synset_id":"consider.v.03"},{"index":4,"sense_key":"consider%2:32:00::","synset_id":"consider.v.04"},{"index":5,"sense_key":"consider%2:32:02::","synset_id":"consider.v.05"},{"index":6,"sense_key":"consider%2:31:08::","synset_id":"think.v.01"},{"index":7,"sense_key":"consider%2:39:04::","synset_id":"regard.v.02"},{"index":8,"sense_key":"consider%2:39:08::","synset_id":"view.v.02"},{"index":9,"sense_key":"consider%2:31:05::","synset_id":"consider.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"consider%2:31:00::","call_ids":["senseval2.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"consider%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037459999999999994,"input_uncached_usd":0.0000354,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.968049707997125}
{"item_id":"senseval2.d000.s017.t010","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:10:00::","synset_id":"english.n.01"},{"index":2,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":3,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":4,"sense_key":"english%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002095,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.938799416995607}
{"item_id":"senseval2.d000.s017.t012","gold_sense_keys":["fit%3:00:02::"],"candidates":[{"index":1,"sense_key":"fit%3:00:02::","synset_id":"fit.a.01"},{"index":2,"sense_key":"fit%5:00:00:ready:00","synset_id":"fit.s.01"},{"index":3,"sense_key":"fit%3:00:01::","synset_id":"fit.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fit%3:00:02::","call_ids":["senseval2.d000.s017.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fit%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054195,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.834207708016038}
{"item_id":"senseval2.d000.s017.t013","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d000.s017.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003191,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.151870000001509}
{"item_id":"senseval2.d000.s017.t014","gold_sense_keys":["foreigner%1:18:00::"],"candidates":[{"index":1,"sense_key":"foreigner%1:18:00::","synset_id":"foreigner.n.01"},{"index":2,"sense_key":"foreigner%1:18:01::","synset_id":"foreigner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foreigner%1:18:00::","call_ids":["senseval2.d000.s017.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foreigner%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036004999999999994,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.833077499992214}
{"item_id":"senseval2.d000.s018.t002","gold_sense_keys":["english%1:18:00::"],"candidates":[{"index":1,"sense_key":"english%1:10:00::","synset_id":"english.n.01"},{"index":2,"sense_key":"english%1:18:00::","synset_id":"english.n.02"},{"index":3,"sense_key":"english%1:09:00::","synset_id":"english.n.03"},{"index":4,"sense_key":"english%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"english%1:18:00::","call_ids":["senseval2.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"english%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002875,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.1582724579493515}
{"item_id":"senseval2.d000.s018.t003","gold_sense_keys":["invent%2:36:00::"],"candidates":[{"index":1,"sense_key":"invent%2:36:00::","synset_id":"invent.v.01"},{"index":2,"sense_key":"invent%2:36:01::","synset_id":"fabricate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"invent%2:36:00::","call_ids":["senseval2.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"invent%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002851,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.94083049998153}
{"item_id":"senseval2.d000.s018.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004944999999999999,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.503285583981778}
{"item_id":"senseval2.d000.s018.t006","gold_sense_keys":["require%2:34:00::","require%2:42:00::"],"candidates":[{"index":1,"sense_key":"require%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"require%2:32:01::","synset_id":"ask.v.04"},{"index":3,"sense_key":"require%2:32:00::","synset_id":"command.v.02"},{"index":4,"sense_key":"require%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"require%2:42:00::","call_ids":["senseval2.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"require%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002626,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.548364666989073}
{"item_id":"senseval2.d000.s018.t007","gold_sense_keys":["physical%3:00:00::"],"candidates":[{"index":1,"sense_key":"physical%3:00:00::","synset_id":"physical.a.01"},{"index":2,"sense_key":"physical%3:01:00::","synset_id":"physical.a.02"},{"index":3,"sense_key":"physical%5:00:00:material:01","synset_id":"physical.s.01"},{"index":4,"sense_key":"physical%5:00:00:natural:03","synset_id":"physical.s.02"},{"index":5,"sense_key":"physical%5:00:00:energetic:00","synset_id":"physical.s.03"},{"index":6,"sense_key":"physical%5:00:00:forceful:00","synset_id":"forcible.s.01"},{"index":7,"sense_key":"physical%5:00:02:material:01","synset_id":"physical.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"physical%3:00:00::","call_ids":["senseval2.d000.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"physical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019164999999999997,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.392385791987181}
{"item_id":"senseval2.d000.s018.t009","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003273,"input_uncached_usd":0.0000389,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.568913750001229}
{"item_id":"senseval2.d000.s018.t010","gold_sense_keys":["weigh%2:42:01::"],"candidates":[{"index":1,"sense_key":"weigh%2:42:01::","synset_id":"weigh.v.01"},{"index":2,"sense_key":"weigh%2:32:00::","synset_id":"consider.v.04"},{"index":3,"sense_key":"weigh%2:42:00::","synset_id":"weigh.v.03"},{"index":4,"sense_key":"weigh%2:42:03::","synset_id":"count.v.02"},{"index":5,"sense_key":"weigh%2:42:04::","synset_id":"weigh.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"weigh%2:42:00::","call_ids":["senseval2.d000.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"weigh%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011034999999999999,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.311543249990791}
{"item_id":"senseval2.d000.s018.t014","gold_sense_keys":["intense%3:00:00::"],"candidates":[{"index":1,"sense_key":"intense%3:00:00::","synset_id":"intense.a.01"},{"index":2,"sense_key":"intense%5:00:00:sharp:04","synset_id":"acute.s.01"},{"index":3,"sense_key":"intense%5:00:00:saturated:03","synset_id":"intense.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intense%3:00:00::","call_ids":["senseval2.d000.s018.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intense%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000286,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.947354499949142}
{"item_id":"senseval2.d000.s018.t015","gold_sense_keys":["mental%3:00:00::","mental%3:01:00::"],"candidates":[{"index":1,"sense_key":"mental%3:00:00::","synset_id":"mental.a.01"},{"index":2,"sense_key":"mental%3:01:00::","synset_id":"mental.a.02"},{"index":3,"sense_key":"mental%3:01:03::","synset_id":"mental.a.03"},{"index":4,"sense_key":"mental%3:01:04::","synset_id":"genial.a.02"},{"index":5,"sense_key":"mental%5:00:00:unhealthy:00","synset_id":"mental.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mental%3:00:00::","call_ids":["senseval2.d000.s018.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mental%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028855,"input_uncached_usd":0.00002615,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.417751083034091}
{"item_id":"senseval2.d000.s018.t016","gold_sense_keys":["concentration%1:07:01::","concentration%1:09:00::"],"candidates":[{"index":1,"sense_key":"concentration%1:07:02::","synset_id":"concentration.n.01"},{"index":2,"sense_key":"concentration%1:07:03::","synset_id":"concentration.n.02"},{"index":3,"sense_key":"concentration%1:04:01::","synset_id":"concentration.n.03"},{"index":4,"sense_key":"concentration%1:11:00::","synset_id":"concentration.n.04"},{"index":5,"sense_key":"concentration%1:09:00::","synset_id":"concentration.n.05"},{"index":6,"sense_key":"concentration%1:04:02::","synset_id":"concentration.n.06"},{"index":7,"sense_key":"concentration%1:07:01::","synset_id":"assiduity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"concentration%1:09:00::","call_ids":["senseval2.d000.s018.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"concentration%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039339999999999997,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.058539041958284}
{"item_id":"senseval2.d000.s019.t000","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:00::","synset_id":"english.a.01"},{"index":2,"sense_key":"english%3:01:01::","synset_id":"english.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s019.t000__v1__a1","senseval2.d000.s019.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":861,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041425,"input_uncached_usd":0.00004305,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.12545825005509}
{"item_id":"senseval2.d000.s019.t001","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004545,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.617031250032596}
{"item_id":"senseval2.d000.s019.t003","gold_sense_keys":["round%1:28:01::"],"candidates":[{"index":1,"sense_key":"round%1:06:01::","synset_id":"round.n.01"},{"index":2,"sense_key":"round%1:28:01::","synset_id":"cycle.n.01"},{"index":3,"sense_key":"round%1:15:00::","synset_id":"beat.n.01"},{"index":4,"sense_key":"round%1:04:00::","synset_id":"round.n.04"},{"index":5,"sense_key":"round%1:04:03::","synset_id":"round_of_golf.n.01"},{"index":6,"sense_key":"round%1:04:01::","synset_id":"round.n.06"},{"index":7,"sense_key":"round%1:28:00::","synset_id":"turn.n.09"},{"index":8,"sense_key":"round%1:17:00::","synset_id":"round.n.08"},{"index":9,"sense_key":"round%1:13:01::","synset_id":"round.n.09"},{"index":10,"sense_key":"round%1:13:00::","synset_id":"round.n.10"},{"index":11,"sense_key":"round%1:10:00::","synset_id":"round.n.11"},{"index":12,"sense_key":"round%1:10:01::","synset_id":"round.n.12"},{"index":13,"sense_key":"round%1:06:03::","synset_id":"rung.n.01"},{"index":14,"sense_key":"round%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"round%1:10:00::","call_ids":["senseval2.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"round%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005115,"input_uncached_usd":0.0000443,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.126859041978605}
{"item_id":"senseval2.d000.s019.t004","gold_sense_keys":["high-pitched%3:00:00::"],"candidates":[{"index":1,"sense_key":"high-pitched%3:00:00::","synset_id":"high.a.04"},{"index":2,"sense_key":"high-pitched%5:00:00:inclined:01","synset_id":"high-pitched.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high-pitched%3:00:00::","call_ids":["senseval2.d000.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high-pitched%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018094999999999998,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9604233750142157}
{"item_id":"senseval2.d000.s019.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001473,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5734505419968627}
{"item_id":"senseval2.d000.s019.t006","gold_sense_keys":["low%3:00:03::"],"candidates":[{"index":1,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":2,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":3,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":4,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":5,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":6,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":7,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":8,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"},{"index":9,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":10,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"low%3:00:03::","call_ids":["senseval2.d000.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"low%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025065,"input_uncached_usd":0.00003945,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.113469249976333}
{"item_id":"senseval2.d000.s019.t007","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"simple%5:00:00:easy:01","call_ids":["senseval2.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"simple%5:00:00:easy:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00062755,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.319464916014113}
{"item_id":"senseval2.d000.s019.t009","gold_sense_keys":["scale%1:10:00::"],"candidates":[{"index":1,"sense_key":"scale%1:24:03::","synset_id":"scale.n.01"},{"index":2,"sense_key":"scale%1:24:00::","synset_id":"scale.n.02"},{"index":3,"sense_key":"scale%1:24:01::","synset_id":"scale.n.03"},{"index":4,"sense_key":"scale%1:20:00::","synset_id":"scale.n.04"},{"index":5,"sense_key":"scale%1:17:00::","synset_id":"scale.n.05"},{"index":6,"sense_key":"scale%1:10:00::","synset_id":"scale.n.06"},{"index":7,"sense_key":"scale%1:06:02::","synset_id":"scale.n.07"},{"index":8,"sense_key":"scale%1:06:00::","synset_id":"scale.n.08"},{"index":9,"sense_key":"scale%1:06:03::","synset_id":"plate.n.14"},{"index":10,"sense_key":"scale%1:05:00::","synset_id":"scale.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"scale%1:10:00::","call_ids":["senseval2.d000.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"scale%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016974999999999998,"input_uncached_usd":0.00003535,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8183558330056258}
{"item_id":"senseval2.d000.s019.t010","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00062635,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.029027291981038}
{"item_id":"senseval2.d000.s019.t012","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001335,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.164016750000883}
{"item_id":"senseval2.d000.s019.t014","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s019.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030084999999999997,"input_uncached_usd":0.00003845,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.738326542021241}
{"item_id":"senseval2.d000.s020.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001837,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.472378415986896}
{"item_id":"senseval2.d000.s020.t001","gold_sense_keys":["signal%1:10:00::","signal%1:16:00::"],"candidates":[{"index":1,"sense_key":"signal%1:10:00::","synset_id":"signal.n.01"},{"index":2,"sense_key":"signal%1:16:00::","synset_id":"signal.n.02"},{"index":3,"sense_key":"signal%1:19:00::","synset_id":"signal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"signal%1:16:00::","call_ids":["senseval2.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"signal%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033719999999999996,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.6933629579725675}
{"item_id":"senseval2.d000.s020.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015859999999999998,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.906523958023172}
{"item_id":"senseval2.d000.s020.t003","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"begin%2:30:01::","call_ids":["senseval2.d000.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"begin%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002539,"input_uncached_usd":0.0000427,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.696752374991775}
{"item_id":"senseval2.d000.s020.t004","gold_sense_keys":["vary%2:30:04::"],"candidates":[{"index":1,"sense_key":"vary%2:30:01::","synset_id":"change.v.03"},{"index":2,"sense_key":"vary%2:42:00::","synset_id":"deviate.v.02"},{"index":3,"sense_key":"vary%2:42:01::","synset_id":"vary.v.03"},{"index":4,"sense_key":"vary%2:30:04::","synset_id":"vary.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vary%2:42:01::","call_ids":["senseval2.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vary%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051915,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.393300624971744}
{"item_id":"senseval2.d000.s020.t005","gold_sense_keys":["order%1:14:00::"],"candidates":[{"index":1,"sense_key":"order%1:10:03::","synset_id":"order.n.01"},{"index":2,"sense_key":"order%1:07:01::","synset_id":"order.n.02"},{"index":3,"sense_key":"order%1:26:00::","synset_id":"order.n.03"},{"index":4,"sense_key":"order%1:14:00::","synset_id":"ordering.n.01"},{"index":5,"sense_key":"order%1:26:02::","synset_id":"orderliness.n.02"},{"index":6,"sense_key":"order%1:10:00::","synset_id":"decree.n.01"},{"index":7,"sense_key":"order%1:10:01::","synset_id":"order.n.07"},{"index":8,"sense_key":"order%1:14:02::","synset_id":"club.n.02"},{"index":9,"sense_key":"order%1:10:02::","synset_id":"order.n.09"},{"index":10,"sense_key":"order%1:26:01::","synset_id":"holy_order.n.01"},{"index":11,"sense_key":"order%1:14:03::","synset_id":"order.n.11"},{"index":12,"sense_key":"order%1:14:01::","synset_id":"order.n.12"},{"index":13,"sense_key":"order%1:10:04::","synset_id":"order.n.13"},{"index":14,"sense_key":"order%1:07:00::","synset_id":"order.n.14"},{"index":15,"sense_key":"order%1:04:00::","synset_id":"order.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"order%1:14:00::","call_ids":["senseval2.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"order%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00021344999999999998,"input_uncached_usd":0.00005345,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3395730829797685}
{"item_id":"senseval2.d000.s020.t006","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002499,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.134040500037372}
{"item_id":"senseval2.d000.s020.t007","gold_sense_keys":["sound%2:39:00::","sound%2:39:03::"],"candidates":[{"index":1,"sense_key":"sound%2:39:06::","synset_id":"sound.v.01"},{"index":2,"sense_key":"sound%2:39:00::","synset_id":"sound.v.02"},{"index":3,"sense_key":"sound%2:39:03::","synset_id":"sound.v.03"},{"index":4,"sense_key":"sound%2:39:02::","synset_id":"sound.v.04"},{"index":5,"sense_key":"sound%2:32:00::","synset_id":"voice.v.02"},{"index":6,"sense_key":"sound%2:39:01::","synset_id":"sound.v.06"},{"index":7,"sense_key":"sound%2:30:00::","synset_id":"fathom.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sound%2:39:03::","call_ids":["senseval2.d000.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sound%2:39:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031675000000000003,"input_uncached_usd":0.000028749999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.138469333003741}
{"item_id":"senseval2.d000.s020.t008","gold_sense_keys":["alter%2:30:01::"],"candidates":[{"index":1,"sense_key":"alter%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"alter%2:30:00::","synset_id":"change.v.03"},{"index":3,"sense_key":"alter%2:36:00::","synset_id":"alter.v.03"},{"index":4,"sense_key":"alter%2:30:04::","synset_id":"interpolate.v.02"},{"index":5,"sense_key":"alter%2:29:00::","synset_id":"alter.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"alter%2:36:00::","call_ids":["senseval2.d000.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"alter%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00072645,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.076543083996512}
{"item_id":"senseval2.d000.s020.t009","gold_sense_keys":["steady%3:00:00::"],"candidates":[{"index":1,"sense_key":"steady%3:00:00::","synset_id":"steady.a.01"},{"index":2,"sense_key":"steady%5:00:00:stable:00","synset_id":"firm.s.06"},{"index":3,"sense_key":"steady%5:00:00:secure:03","synset_id":"steady.s.02"},{"index":4,"sense_key":"steady%5:00:00:resolute:00","synset_id":"firm.s.01"},{"index":5,"sense_key":"steady%5:00:00:frequent:00","synset_id":"regular.s.06"},{"index":6,"sense_key":"steady%5:00:00:unexcitable:00","synset_id":"steady.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"steady%3:00:00::","call_ids":["senseval2.d000.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"steady%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026705,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.85407420800766}
{"item_id":"senseval2.d000.s020.t010","gold_sense_keys":["rhythm%1:10:01::"],"candidates":[{"index":1,"sense_key":"rhythm%1:10:01::","synset_id":"rhythm.n.01"},{"index":2,"sense_key":"rhythm%1:07:01::","synset_id":"rhythm.n.02"},{"index":3,"sense_key":"rhythm%1:28:00::","synset_id":"cycle.n.01"},{"index":4,"sense_key":"rhythm%1:10:00::","synset_id":"rhythm.n.04"},{"index":5,"sense_key":"rhythm%1:04:00::","synset_id":"rhythm_method_of_birth_control.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rhythm%1:10:01::","call_ids":["senseval2.d000.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rhythm%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036884999999999994,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.14236920804251}
{"item_id":"senseval2.d000.s020.t011","gold_sense_keys":["striking%1:04:00::"],"candidates":[{"index":1,"sense_key":"striking%1:11:00::","synset_id":"contact.n.04"},{"index":2,"sense_key":"striking%1:04:00::","synset_id":"hit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"striking%1:04:00::","call_ids":["senseval2.d000.s020.t011__v1__a1","senseval2.d000.s020.t011__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"striking%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":929,"cached_input_tokens":0,"output_tokens":2144,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00090405,"input_uncached_usd":0.000046450000000000004,"input_cached_usd":0.0,"output_usd":0.0008576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.18831970897736}
{"item_id":"senseval2.d000.s021.t000","gold_sense_keys":["variation%1:10:00::"],"candidates":[{"index":1,"sense_key":"variation%1:11:01::","synset_id":"variation.n.01"},{"index":2,"sense_key":"variation%1:04:00::","synset_id":"variation.n.02"},{"index":3,"sense_key":"variation%1:10:00::","synset_id":"variation.n.03"},{"index":4,"sense_key":"variation%1:09:01::","synset_id":"version.n.02"},{"index":5,"sense_key":"variation%1:06:00::","synset_id":"variation.n.05"},{"index":6,"sense_key":"variation%1:25:00::","synset_id":"magnetic_declination.n.01"},{"index":7,"sense_key":"variation%1:22:00::","synset_id":"variation.n.07"},{"index":8,"sense_key":"variation%1:19:00::","synset_id":"variation.n.08"},{"index":9,"sense_key":"variation%1:18:00::","synset_id":"mutant.n.01"},{"index":10,"sense_key":"variation%1:04:02::","synset_id":"pas_seul.n.01"},{"index":11,"sense_key":"variation%1:04:01::","synset_id":"variation.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"variation%1:11:01::","call_ids":["senseval2.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"variation%1:11:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005819,"input_uncached_usd":0.0000379,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.418837041012011}
{"item_id":"senseval2.d000.s021.t002","gold_sense_keys":["occur%2:30:00::"],"candidates":[{"index":1,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"},{"index":3,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"occur%2:30:00::","call_ids":["senseval2.d000.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"occur%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020999999999999998,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.500357916986104}
{"item_id":"senseval2.d000.s021.t003","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d000.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044724999999999995,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.067167458008043}
{"item_id":"senseval2.d000.s021.t004","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:00::","call_ids":["senseval2.d000.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020859999999999997,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.2070039579994045}
{"item_id":"senseval2.d000.s021.t005","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["senseval2.d000.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038399999999999996,"input_uncached_usd":0.0000448,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.386354083020706}
{"item_id":"senseval2.d000.s021.t006","gold_sense_keys":["state%2:32:00::"],"candidates":[{"index":1,"sense_key":"state%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"state%2:32:01::","synset_id":"submit.v.02"},{"index":3,"sense_key":"state%2:32:02::","synset_id":"express.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%2:32:00::","call_ids":["senseval2.d000.s021.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018345,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.480740749975666}
{"item_id":"senseval2.d000.s022.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013335,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.797351000015624}
{"item_id":"senseval2.d000.s022.t005","gold_sense_keys":["method%1:09:00::"],"candidates":[{"index":1,"sense_key":"method%1:09:00::","synset_id":"method.n.01"},{"index":2,"sense_key":"method%1:04:00::","synset_id":"method_acting.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"method%1:09:00::","call_ids":["senseval2.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"method%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033605,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.895298666029703}
{"item_id":"senseval2.d000.s022.t006","gold_sense_keys":["name%1:10:00::"],"candidates":[{"index":1,"sense_key":"name%1:10:00::","synset_id":"name.n.01"},{"index":2,"sense_key":"name%1:26:00::","synset_id":"name.n.02"},{"index":3,"sense_key":"name%1:14:00::","synset_id":"name.n.03"},{"index":4,"sense_key":"name%1:18:00::","synset_id":"name.n.04"},{"index":5,"sense_key":"name%1:04:00::","synset_id":"name.n.05"},{"index":6,"sense_key":"name%1:10:01::","synset_id":"name.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%1:10:00::","call_ids":["senseval2.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003425,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.168344666948542}
{"item_id":"senseval2.d000.s023.t000","gold_sense_keys":["series%1:14:00::"],"candidates":[{"index":1,"sense_key":"series%1:14:00::","synset_id":"series.n.01"},{"index":2,"sense_key":"series%1:10:01::","synset_id":"serial.n.01"},{"index":3,"sense_key":"series%1:10:00::","synset_id":"series.n.03"},{"index":4,"sense_key":"series%1:11:00::","synset_id":"series.n.04"},{"index":5,"sense_key":"series%1:24:00::","synset_id":"series.n.05"},{"index":6,"sense_key":"series%1:14:01::","synset_id":"series.n.06"},{"index":7,"sense_key":"series%1:09:00::","synset_id":"series.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"series%1:14:00::","call_ids":["senseval2.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"series%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021754999999999998,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.192626792006195}
{"item_id":"senseval2.d000.s023.t003","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval2.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2211,"cached_input_tokens":0,"output_tokens":1937,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008853499999999999,"input_uncached_usd":0.00011054999999999999,"input_cached_usd":0.0,"output_usd":0.0007748,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.047140208014753}
{"item_id":"senseval2.d000.s023.t004","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016795,"input_uncached_usd":0.000033549999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.034706374979578}
{"item_id":"senseval2.d000.s023.t005","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval2.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018519999999999998,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.897378875000868}
{"item_id":"senseval2.d000.s024.t000","gold_sense_keys":["look%1:04:00::"],"candidates":[{"index":1,"sense_key":"look%1:07:01::","synset_id":"expression.n.01"},{"index":2,"sense_key":"look%1:04:00::","synset_id":"look.n.02"},{"index":3,"sense_key":"look%1:07:00::","synset_id":"look.n.03"},{"index":4,"sense_key":"look%1:26:00::","synset_id":"spirit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%1:04:00::","call_ids":["senseval2.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016214999999999998,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.083140917005949}
{"item_id":"senseval2.d000.s024.t002","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"night%1:28:03::","call_ids":["senseval2.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"night%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034395,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.674909416004084}
{"item_id":"senseval2.d000.s024.t003","gold_sense_keys":["practice%1:04:02::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"practice%1:04:02::","call_ids":["senseval2.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"practice%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026504999999999996,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.5061757089570165}
{"item_id":"senseval2.d000.s024.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002878,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.621954999980517}
{"item_id":"senseval2.d000.s024.t006","gold_sense_keys":["give%2:32:08::","give%2:40:03::","give%2:40:05::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval2.d000.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"give%2:40:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2106,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004701,"input_uncached_usd":0.0001053,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.491338791965973}
{"item_id":"senseval2.d000.s024.t007","gold_sense_keys":["idea%1:09:02::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"idea%1:09:02::","call_ids":["senseval2.d000.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"idea%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039479999999999995,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.515119959018193}
{"item_id":"senseval2.d000.s024.t008","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["senseval2.d000.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00078465,"input_uncached_usd":0.00003585,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.256697374978103}
{"item_id":"senseval2.d000.s024.t009","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d000.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004969,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.711732790980022}
{"item_id":"senseval2.d000.s025.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026095,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.045696749992203}
{"item_id":"senseval2.d000.s025.t001","gold_sense_keys":["stand%2:35:00::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stand%2:42:03::","call_ids":["senseval2.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stand%2:42:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014895,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7726307500270195}
{"item_id":"senseval2.d000.s025.t002","gold_sense_keys":["circle%1:25:01::"],"candidates":[{"index":1,"sense_key":"circle%1:25:00::","synset_id":"circle.n.01"},{"index":2,"sense_key":"circle%1:14:00::","synset_id":"set.n.05"},{"index":3,"sense_key":"circle%1:25:01::","synset_id":"circle.n.03"},{"index":4,"sense_key":"circle%1:04:00::","synset_id":"lap.n.05"},{"index":5,"sense_key":"circle%1:06:01::","synset_id":"traffic_circle.n.01"},{"index":6,"sense_key":"circle%1:06:03::","synset_id":"r-2.n.01"},{"index":7,"sense_key":"circle%1:06:02::","synset_id":"circle.n.07"},{"index":8,"sense_key":"circle%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"circle%1:25:01::","call_ids":["senseval2.d000.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"circle%1:25:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014395,"input_uncached_usd":0.00003515,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4424705000128597}
{"item_id":"senseval2.d000.s025.t003","gold_sense_keys":["foot%1:08:01::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foot%1:08:01::","call_ids":["senseval2.d000.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foot%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040494999999999995,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.7246008749934845}
{"item_id":"senseval2.d000.s025.t004","gold_sense_keys":["ahead%4:02:00::"],"candidates":[{"index":1,"sense_key":"ahead%4:02:00::","synset_id":"ahead.r.01"},{"index":2,"sense_key":"ahead%4:02:06::","synset_id":"ahead.r.02"},{"index":3,"sense_key":"ahead%4:02:02::","synset_id":"ahead.r.03"},{"index":4,"sense_key":"ahead%4:02:01::","synset_id":"ahead.r.04"},{"index":5,"sense_key":"ahead%4:02:05::","synset_id":"ahead.r.05"},{"index":6,"sense_key":"ahead%4:02:04::","synset_id":"ahead.r.06"},{"index":7,"sense_key":"ahead%4:02:03::","synset_id":"ahead.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ahead%4:02:02::","call_ids":["senseval2.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ahead%4:02:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014314999999999998,"input_uncached_usd":0.00003435,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4136749579920433}
{"item_id":"senseval2.d000.s025.t006","gold_sense_keys":["stance%1:07:00::"],"candidates":[{"index":1,"sense_key":"stance%1:07:00::","synset_id":"stance.n.01"},{"index":2,"sense_key":"stance%1:09:00::","synset_id":"position.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stance%1:07:00::","call_ids":["senseval2.d000.s025.t006__v1__a1","senseval2.d000.s025.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stance%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00054055,"input_uncached_usd":0.00004135,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.80406579200644}
{"item_id":"senseval2.d000.s025.t007","gold_sense_keys":["pull%2:35:04::"],"candidates":[{"index":1,"sense_key":"pull%2:35:00::","synset_id":"pull.v.01"},{"index":2,"sense_key":"pull%2:35:02::","synset_id":"attract.v.01"},{"index":3,"sense_key":"pull%2:38:01::","synset_id":"pull.v.03"},{"index":4,"sense_key":"pull%2:35:04::","synset_id":"pull.v.04"},{"index":5,"sense_key":"pull%2:41:00::","synset_id":"perpetrate.v.01"},{"index":6,"sense_key":"pull%2:38:03::","synset_id":"draw.v.05"},{"index":7,"sense_key":"pull%2:38:00::","synset_id":"pull.v.07"},{"index":8,"sense_key":"pull%2:29:00::","synset_id":"pull.v.08"},{"index":9,"sense_key":"pull%2:38:13::","synset_id":"pull.v.09"},{"index":10,"sense_key":"pull%2:38:02::","synset_id":"pull.v.10"},{"index":11,"sense_key":"pull%2:38:04::","synset_id":"pull.v.11"},{"index":12,"sense_key":"pull%2:35:01::","synset_id":"rend.v.01"},{"index":13,"sense_key":"pull%2:35:03::","synset_id":"pull.v.13"},{"index":14,"sense_key":"pull%2:35:05::","synset_id":"pluck.v.05"},{"index":15,"sense_key":"pull%2:35:10::","synset_id":"extract.v.01"},{"index":16,"sense_key":"pull%2:33:13::","synset_id":"pull.v.16"},{"index":17,"sense_key":"pull%2:30:11::","synset_id":"pull.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pull%2:35:04::","call_ids":["senseval2.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pull%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1101,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00024104999999999998,"input_uncached_usd":0.000055049999999999996,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3488218749989755}
{"item_id":"senseval2.d000.s025.t008","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s025.t008__v1__a1","senseval2.d000.s025.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056835,"input_uncached_usd":0.00004355,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.356720833980944}
{"item_id":"senseval2.d000.s025.t009","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"small%5:00:00:limited:00","call_ids":["senseval2.d000.s025.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"small%5:00:00:limited:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":1681,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007115499999999998,"input_uncached_usd":0.000039149999999999996,"input_cached_usd":0.0,"output_usd":0.0006724,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.122555957990699}
{"item_id":"senseval2.d000.s025.t010","gold_sense_keys":["hole%1:06:00::","hole%1:17:01::"],"candidates":[{"index":1,"sense_key":"hole%1:17:01::","synset_id":"hole.n.01"},{"index":2,"sense_key":"hole%1:06:00::","synset_id":"hole.n.02"},{"index":3,"sense_key":"hole%1:06:01::","synset_id":"hole.n.03"},{"index":4,"sense_key":"hole%1:25:00::","synset_id":"hole.n.04"},{"index":5,"sense_key":"hole%1:17:02::","synset_id":"hole.n.05"},{"index":6,"sense_key":"hole%1:26:01::","synset_id":"hole.n.06"},{"index":7,"sense_key":"hole%1:26:00::","synset_id":"fix.n.01"},{"index":8,"sense_key":"hole%1:08:00::","synset_id":"trap.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hole%1:06:00::","call_ids":["senseval2.d000.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hole%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031835,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.1513528330251575}
{"item_id":"senseval2.d000.s025.t011","gold_sense_keys":["high%3:00:01::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"high%3:00:01::","call_ids":["senseval2.d000.s025.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"high%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039634999999999996,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.65993999998318}
{"item_id":"senseval2.d000.s025.t012","gold_sense_keys":["ceiling%1:06:00::"],"candidates":[{"index":1,"sense_key":"ceiling%1:06:00::","synset_id":"ceiling.n.01"},{"index":2,"sense_key":"ceiling%1:07:00::","synset_id":"ceiling.n.02"},{"index":3,"sense_key":"ceiling%1:10:00::","synset_id":"ceiling.n.03"},{"index":4,"sense_key":"ceiling%1:07:01::","synset_id":"ceiling.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ceiling%1:06:00::","call_ids":["senseval2.d000.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ceiling%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046619999999999995,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.209307000041008}
{"item_id":"senseval2.d000.s025.t013","gold_sense_keys":["chamber%1:06:00::"],"candidates":[{"index":1,"sense_key":"chamber%1:06:00::","synset_id":"chamber.n.01"},{"index":2,"sense_key":"chamber%1:08:00::","synset_id":"chamber.n.02"},{"index":3,"sense_key":"chamber%1:06:02::","synset_id":"chamber.n.03"},{"index":4,"sense_key":"chamber%1:14:00::","synset_id":"chamber.n.04"},{"index":5,"sense_key":"chamber%1:06:01::","synset_id":"bedroom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chamber%1:06:00::","call_ids":["senseval2.d000.s025.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chamber%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049265,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.628143250010908}
{"item_id":"senseval2.d000.s026.t000","gold_sense_keys":["speak%2:32:02::"],"candidates":[{"index":1,"sense_key":"speak%2:32:00::","synset_id":"talk.v.02"},{"index":2,"sense_key":"speak%2:32:01::","synset_id":"talk.v.01"},{"index":3,"sense_key":"speak%2:32:02::","synset_id":"speak.v.03"},{"index":4,"sense_key":"speak%2:32:03::","synset_id":"address.v.02"},{"index":5,"sense_key":"speak%2:39:00::","synset_id":"speak.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"speak%2:32:00::","call_ids":["senseval2.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"speak%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.000443,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.703145042003598}
{"item_id":"senseval2.d000.s026.t002","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038574999999999997,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.621836250007618}
{"item_id":"senseval2.d000.s026.t003","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval2.d000.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006987499999999999,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.903292417002376}
{"item_id":"senseval2.d000.s026.t004","gold_sense_keys":["make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval2.d000.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2308,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0005314,"input_uncached_usd":0.0001154,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.40867558302125}
{"item_id":"senseval2.d000.s026.t006","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019324999999999998,"input_uncached_usd":0.00003325,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.404147499997634}
{"item_id":"senseval2.d000.s026.t007","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032695,"input_uncached_usd":0.00003895,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.216257166001014}
{"item_id":"senseval2.d000.s026.t008","gold_sense_keys":["muffle%2:39:00::"],"candidates":[{"index":1,"sense_key":"muffle%2:35:00::","synset_id":"smother.v.03"},{"index":2,"sense_key":"muffle%2:39:00::","synset_id":"muffle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"muffle%2:39:00::","call_ids":["senseval2.d000.s026.t008__v1__a1","senseval2.d000.s026.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"muffle%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":1567,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00067335,"input_uncached_usd":0.00004654999999999999,"input_cached_usd":0.0,"output_usd":0.0006268,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.338309291983023}
{"item_id":"senseval2.d000.s026.t009","gold_sense_keys":["ceiling%1:06:00::"],"candidates":[{"index":1,"sense_key":"ceiling%1:06:00::","synset_id":"ceiling.n.01"},{"index":2,"sense_key":"ceiling%1:07:00::","synset_id":"ceiling.n.02"},{"index":3,"sense_key":"ceiling%1:10:00::","synset_id":"ceiling.n.03"},{"index":4,"sense_key":"ceiling%1:07:01::","synset_id":"ceiling.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ceiling%1:06:00::","call_ids":["senseval2.d000.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ceiling%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013365,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5961599170113914}
{"item_id":"senseval2.d000.s027.t001","gold_sense_keys":["absorbed%5:00:00:attentive:00"],"candidates":[{"index":1,"sense_key":"absorbed%5:00:00:attentive:00","synset_id":"captive.s.02"},{"index":2,"sense_key":"absorbed%5:00:00:unreflected:00","synset_id":"absorbed.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"absorbed%5:00:00:attentive:00","call_ids":["senseval2.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"absorbed%5:00:00:attentive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018399999999999997,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.10603408399038}
{"item_id":"senseval2.d000.s027.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001862,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.567637917003594}
{"item_id":"senseval2.d000.s027.t003","gold_sense_keys":["stare%2:29:00::"],"candidates":[{"index":1,"sense_key":"stare%2:39:00::","synset_id":"gaze.v.01"},{"index":2,"sense_key":"stare%2:29:00::","synset_id":"stare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stare%2:39:00::","call_ids":["senseval2.d000.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stare%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003875,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.063971417024732}
{"item_id":"senseval2.d000.s027.t004","gold_sense_keys":["straight%4:02:05::"],"candidates":[{"index":1,"sense_key":"straight%4:02:05::","synset_id":"directly.r.01"},{"index":2,"sense_key":"straight%4:02:04::","synset_id":"directly.r.04"},{"index":3,"sense_key":"straight%4:02:00::","synset_id":"straight.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"straight%4:02:00::","call_ids":["senseval2.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"straight%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026169999999999996,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.56783929199446}
{"item_id":"senseval2.d000.s027.t005","gold_sense_keys":["ahead%4:02:00::"],"candidates":[{"index":1,"sense_key":"ahead%4:02:00::","synset_id":"ahead.r.01"},{"index":2,"sense_key":"ahead%4:02:06::","synset_id":"ahead.r.02"},{"index":3,"sense_key":"ahead%4:02:02::","synset_id":"ahead.r.03"},{"index":4,"sense_key":"ahead%4:02:01::","synset_id":"ahead.r.04"},{"index":5,"sense_key":"ahead%4:02:05::","synset_id":"ahead.r.05"},{"index":6,"sense_key":"ahead%4:02:04::","synset_id":"ahead.r.06"},{"index":7,"sense_key":"ahead%4:02:03::","synset_id":"ahead.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ahead%4:02:00::","call_ids":["senseval2.d000.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ahead%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00068235,"input_uncached_usd":0.00003595,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.121215542021673}
{"item_id":"senseval2.d000.s027.t006","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000321,"input_uncached_usd":0.000032999999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.493902665970381}
{"item_id":"senseval2.d000.s027.t008","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"call%2:32:00::","call_ids":["senseval2.d000.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"call%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1556,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00039139999999999997,"input_uncached_usd":0.0000778,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.53408833296271}
{"item_id":"senseval2.d000.s027.t009","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval2.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045034999999999997,"input_uncached_usd":0.00003435,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.046757208008785}
{"item_id":"senseval2.d000.s027.t010","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d000.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003924,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.981976583017968}
{"item_id":"senseval2.d000.s027.t011","gold_sense_keys":["rope%1:06:00::"],"candidates":[{"index":1,"sense_key":"rope%1:06:00::","synset_id":"rope.n.01"},{"index":2,"sense_key":"rope%1:06:01::","synset_id":"r-2.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rope%1:06:00::","call_ids":["senseval2.d000.s027.t011__v1__a1","senseval2.d000.s027.t011__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rope%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041814999999999995,"input_uncached_usd":0.00004695,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.597588707925752}
{"item_id":"senseval2.d000.s027.t012","gold_sense_keys":["thus%4:02:00::"],"candidates":[{"index":1,"sense_key":"thus%4:02:00::","synset_id":"therefore.r.01"},{"index":2,"sense_key":"thus%4:02:01::","synset_id":"thus.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thus%4:02:01::","call_ids":["senseval2.d000.s027.t012__v1__a1","senseval2.d000.s027.t012__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thus%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1033,"cached_input_tokens":0,"output_tokens":2400,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.00101165,"input_uncached_usd":0.000051649999999999995,"input_cached_usd":0.0,"output_usd":0.0009599999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.87405441602459}
{"item_id":"senseval2.d000.s027.t013","gold_sense_keys":["time%2:30:06::"],"candidates":[{"index":1,"sense_key":"time%2:30:00::","synset_id":"clock.v.01"},{"index":2,"sense_key":"time%2:31:00::","synset_id":"time.v.02"},{"index":3,"sense_key":"time%2:31:01::","synset_id":"time.v.03"},{"index":4,"sense_key":"time%2:30:01::","synset_id":"time.v.04"},{"index":5,"sense_key":"time%2:30:06::","synset_id":"time.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"time%2:30:06::","call_ids":["senseval2.d000.s027.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"time%2:30:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005985500000000001,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.607068250013981}
{"item_id":"senseval2.d000.s027.t014","gold_sense_keys":["pull%1:04:00::"],"candidates":[{"index":1,"sense_key":"pull%1:04:00::","synset_id":"pull.n.01"},{"index":2,"sense_key":"pull%1:19:00::","synset_id":"pull.n.02"},{"index":3,"sense_key":"pull%1:07:00::","synset_id":"pull.n.03"},{"index":4,"sense_key":"pull%1:06:01::","synset_id":"pull.n.04"},{"index":5,"sense_key":"pull%1:26:00::","synset_id":"wrench.n.01"},{"index":6,"sense_key":"pull%1:04:01::","synset_id":"puff.n.07"},{"index":7,"sense_key":"pull%1:04:02::","synset_id":"pull.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pull%1:04:00::","call_ids":["senseval2.d000.s027.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pull%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052665,"input_uncached_usd":0.000033849999999999996,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.423521457996685}
{"item_id":"senseval2.d000.s028.t000","gold_sense_keys":["far%4:02:00::"],"candidates":[{"index":1,"sense_key":"far%4:02:02::","synset_id":"far.r.01"},{"index":2,"sense_key":"far%4:02:00::","synset_id":"far.r.02"},{"index":3,"sense_key":"far%4:02:04::","synset_id":"far.r.03"},{"index":4,"sense_key":"far%4:02:01::","synset_id":"far.r.04"},{"index":5,"sense_key":"far%4:02:05::","synset_id":"far.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"far%4:02:00::","call_ids":["senseval2.d000.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"far%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003424,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.093284957983997}
{"item_id":"senseval2.d000.s028.t001","gold_sense_keys":["above%4:02:01::"],"candidates":[{"index":1,"sense_key":"above%4:02:00::","synset_id":"above.r.01"},{"index":2,"sense_key":"above%4:02:01::","synset_id":"above.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"above%4:02:01::","call_ids":["senseval2.d000.s028.t001__v1__a1","senseval2.d000.s028.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"above%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":2208,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009298500000000001,"input_uncached_usd":0.000046649999999999996,"input_cached_usd":0.0,"output_usd":0.0008832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.241646207985468}
{"item_id":"senseval2.d000.s028.t004","gold_sense_keys":["bronze%5:00:00:metallic:00"],"candidates":[{"index":1,"sense_key":"bronze%5:00:00:chromatic:00","synset_id":"bronze.s.01"},{"index":2,"sense_key":"bronze%5:00:00:metallic:00","synset_id":"bronze.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bronze%5:00:00:metallic:00","call_ids":["senseval2.d000.s028.t004__v1__a1","senseval2.d000.s028.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bronze%5:00:00:metallic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005706499999999999,"input_uncached_usd":0.00004585,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.883271749946289}
{"item_id":"senseval2.d000.s028.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035364999999999995,"input_uncached_usd":0.00004005,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.887436209013686}
{"item_id":"senseval2.d000.s028.t006","gold_sense_keys":["mount%2:35:00::"],"candidates":[{"index":1,"sense_key":"mount%2:35:00::","synset_id":"mount.v.01"},{"index":2,"sense_key":"mount%2:30:00::","synset_id":"wax.v.02"},{"index":3,"sense_key":"mount%2:30:01::","synset_id":"mount.v.03"},{"index":4,"sense_key":"mount%2:36:00::","synset_id":"mount.v.04"},{"index":5,"sense_key":"mount%2:38:01::","synset_id":"hop_on.v.01"},{"index":6,"sense_key":"mount%2:38:00::","synset_id":"climb.v.01"},{"index":7,"sense_key":"mount%2:36:01::","synset_id":"mount.v.07"},{"index":8,"sense_key":"mount%2:35:07::","synset_id":"ride.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mount%2:30:01::","call_ids":["senseval2.d000.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mount%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003989,"input_uncached_usd":0.000034099999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.302556166949216}
{"item_id":"senseval2.d000.s028.t007","gold_sense_keys":["wheel%1:06:00::"],"candidates":[{"index":1,"sense_key":"wheel%1:06:00::","synset_id":"wheel.n.01"},{"index":2,"sense_key":"wheel%1:06:05::","synset_id":"steering_wheel.n.01"},{"index":3,"sense_key":"wheel%1:07:00::","synset_id":"wheel.n.03"},{"index":4,"sense_key":"wheel%1:06:01::","synset_id":"wheel.n.04"},{"index":5,"sense_key":"wheel%1:06:03::","synset_id":"roulette_wheel.n.01"},{"index":6,"sense_key":"wheel%1:06:02::","synset_id":"rack.n.04"},{"index":7,"sense_key":"wheel%1:06:06::","synset_id":"bicycle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wheel%1:06:00::","call_ids":["senseval2.d000.s028.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wheel%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042255,"input_uncached_usd":0.00003175,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.41182266594842}
{"item_id":"senseval2.d000.s028.t008","gold_sense_keys":["swing%2:38:03::"],"candidates":[{"index":1,"sense_key":"swing%2:38:02::","synset_id":"swing.v.01"},{"index":2,"sense_key":"swing%2:38:00::","synset_id":"swing.v.02"},{"index":3,"sense_key":"swing%2:38:03::","synset_id":"swing.v.03"},{"index":4,"sense_key":"swing%2:41:01::","synset_id":"swing.v.04"},{"index":5,"sense_key":"swing%2:35:00::","synset_id":"swing.v.05"},{"index":6,"sense_key":"swing%2:42:00::","synset_id":"dangle.v.01"},{"index":7,"sense_key":"swing%2:33:00::","synset_id":"swing.v.07"},{"index":8,"sense_key":"swing%2:30:08::","synset_id":"swing.v.08"},{"index":9,"sense_key":"swing%2:42:01::","synset_id":"swing.v.09"},{"index":10,"sense_key":"swing%2:42:10::","synset_id":"swing.v.10"},{"index":11,"sense_key":"swing%2:41:00::","synset_id":"swing.v.11"},{"index":12,"sense_key":"swing%2:36:03::","synset_id":"swing.v.12"},{"index":13,"sense_key":"swing%2:35:08::","synset_id":"swing.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swing%2:38:02::","call_ids":["senseval2.d000.s028.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swing%2:38:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00063745,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.59638104098849}
{"item_id":"senseval2.d000.s028.t009","gold_sense_keys":["madly%4:02:02::"],"candidates":[{"index":1,"sense_key":"madly%4:02:02::","synset_id":"madly.r.01"},{"index":2,"sense_key":"madly%4:02:01::","synset_id":"insanely.r.01"},{"index":3,"sense_key":"madly%4:02:00::","synset_id":"madly.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"madly%4:02:02::","call_ids":["senseval2.d000.s028.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"madly%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004416,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.208654124988243}
{"item_id":"senseval2.d000.s028.t010","gold_sense_keys":["full%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"full%5:00:00:complete:00","call_ids":["senseval2.d000.s028.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"full%5:00:00:complete:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00060455,"input_uncached_usd":0.000034949999999999996,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.567028833960649}
{"item_id":"senseval2.d000.s028.t011","gold_sense_keys":["degree%1:23:00::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"degree%1:23:00::","call_ids":["senseval2.d000.s028.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"degree%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011704999999999999,"input_uncached_usd":0.000033849999999999996,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5555219579837285}
{"item_id":"senseval2.d000.s028.t012","gold_sense_keys":["start%2:42:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval2.d000.s028.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00061965,"input_uncached_usd":0.00005005,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.664001083991025}
{"item_id":"senseval2.d000.s028.t013","gold_sense_keys":["end%2:42:00::"],"candidates":[{"index":1,"sense_key":"end%2:42:00::","synset_id":"end.v.01"},{"index":2,"sense_key":"end%2:30:01::","synset_id":"end.v.02"},{"index":3,"sense_key":"end%2:42:01::","synset_id":"end.v.03"},{"index":4,"sense_key":"end%2:36:13::","synset_id":"end.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"end%2:42:01::","call_ids":["senseval2.d000.s028.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"end%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029125,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.505488833005074}
{"item_id":"senseval2.d000.s028.t015","gold_sense_keys":["inverted%5:00:00:turned:00"],"candidates":[{"index":1,"sense_key":"inverted%5:00:00:turned:00","synset_id":"inverted.s.01"},{"index":2,"sense_key":"inverted%3:00:04::","synset_id":"anatropous.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inverted%5:00:00:turned:00","call_ids":["senseval2.d000.s028.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inverted%5:00:00:turned:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023454999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.01670641702367}
{"item_id":"senseval2.d000.s028.t016","gold_sense_keys":["position%1:07:00::"],"candidates":[{"index":1,"sense_key":"position%1:15:00::","synset_id":"position.n.01"},{"index":2,"sense_key":"position%1:15:02::","synset_id":"military_position.n.01"},{"index":3,"sense_key":"position%1:09:00::","synset_id":"position.n.03"},{"index":4,"sense_key":"position%1:07:01::","synset_id":"position.n.04"},{"index":5,"sense_key":"position%1:26:00::","synset_id":"status.n.01"},{"index":6,"sense_key":"position%1:04:01::","synset_id":"position.n.06"},{"index":7,"sense_key":"position%1:07:00::","synset_id":"position.n.07"},{"index":8,"sense_key":"position%1:15:03::","synset_id":"position.n.08"},{"index":9,"sense_key":"position%1:04:02::","synset_id":"position.n.09"},{"index":10,"sense_key":"position%1:04:00::","synset_id":"placement.n.03"},{"index":11,"sense_key":"position%1:26:02::","synset_id":"situation.n.02"},{"index":12,"sense_key":"position%1:09:01::","synset_id":"position.n.12"},{"index":13,"sense_key":"position%1:10:00::","synset_id":"side.n.10"},{"index":14,"sense_key":"position%1:10:01::","synset_id":"place.n.15"},{"index":15,"sense_key":"position%1:04:03::","synset_id":"stead.n.01"},{"index":16,"sense_key":"position%1:04:04::","synset_id":"position.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"position%1:07:00::","call_ids":["senseval2.d000.s028.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"position%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00021214999999999998,"input_uncached_usd":0.00005215,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.277894916012883}
{"item_id":"senseval2.d000.s029.t001","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021189999999999997,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.962994500005152}
{"item_id":"senseval2.d000.s029.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003471,"input_uncached_usd":0.0000335,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.527816125017125}
{"item_id":"senseval2.d000.s029.t005","gold_sense_keys":["retard%2:30:01::"],"candidates":[{"index":1,"sense_key":"retard%2:30:01::","synset_id":"retard.v.01"},{"index":2,"sense_key":"retard%2:42:00::","synset_id":"retard.v.02"},{"index":3,"sense_key":"retard%2:30:07::","synset_id":"check.v.07"},{"index":4,"sense_key":"retard%2:30:00::","synset_id":"decelerate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"retard%2:30:01::","call_ids":["senseval2.d000.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"retard%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002892,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.975517500017304}
{"item_id":"senseval2.d000.s029.t006","gold_sense_keys":["swing%1:04:00::"],"candidates":[{"index":1,"sense_key":"swing%1:26:01::","synset_id":"swing.n.01"},{"index":2,"sense_key":"swing%1:06:00::","synset_id":"swing.n.02"},{"index":3,"sense_key":"swing%1:04:06::","synset_id":"swing.n.03"},{"index":4,"sense_key":"swing%1:04:00::","synset_id":"swing.n.04"},{"index":5,"sense_key":"swing%1:10:01::","synset_id":"swing.n.05"},{"index":6,"sense_key":"swing%1:07:00::","synset_id":"lilt.n.01"},{"index":7,"sense_key":"swing%1:04:03::","synset_id":"golf_stroke.n.01"},{"index":8,"sense_key":"swing%1:04:02::","synset_id":"baseball_swing.n.01"},{"index":9,"sense_key":"swing%1:04:04::","synset_id":"swing.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"swing%1:04:00::","call_ids":["senseval2.d000.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"swing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006317,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.234443291963544}
{"item_id":"senseval2.d000.s029.t007","gold_sense_keys":["so%4:02:06::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"so%4:02:06::","call_ids":["senseval2.d000.s029.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"so%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007155499999999999,"input_uncached_usd":0.000043549999999999995,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.030627958010882}
{"item_id":"senseval2.d000.s029.t008","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s029.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003288,"input_uncached_usd":0.000040799999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.630244500003755}
{"item_id":"senseval2.d000.s029.t009","gold_sense_keys":["swap%2:40:00::"],"candidates":[{"index":1,"sense_key":"swap%2:40:00::","synset_id":"trade.v.04"},{"index":2,"sense_key":"swap%2:38:00::","synset_id":"swap.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swap%2:40:00::","call_ids":["senseval2.d000.s029.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swap%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004131,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.884314541006461}
{"item_id":"senseval2.d000.s029.t010","gold_sense_keys":["place%1:10:02::"],"candidates":[{"index":1,"sense_key":"place%1:15:00::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"place%1:15:04::","synset_id":"place.n.02"},{"index":3,"sense_key":"place%1:09:00::","synset_id":"place.n.03"},{"index":4,"sense_key":"place%1:15:01::","synset_id":"place.n.04"},{"index":5,"sense_key":"place%1:04:01::","synset_id":"stead.n.01"},{"index":6,"sense_key":"place%1:26:01::","synset_id":"place.n.06"},{"index":7,"sense_key":"place%1:15:06::","synset_id":"home.n.01"},{"index":8,"sense_key":"place%1:04:00::","synset_id":"position.n.06"},{"index":9,"sense_key":"place%1:15:05::","synset_id":"position.n.01"},{"index":10,"sense_key":"place%1:26:00::","synset_id":"place.n.10"},{"index":11,"sense_key":"place%1:15:03::","synset_id":"seat.n.01"},{"index":12,"sense_key":"place%1:10:00::","synset_id":"place.n.12"},{"index":13,"sense_key":"place%1:26:02::","synset_id":"place.n.13"},{"index":14,"sense_key":"place%1:15:02::","synset_id":"plaza.n.01"},{"index":15,"sense_key":"place%1:10:02::","synset_id":"place.n.15"},{"index":16,"sense_key":"place%1:10:01::","synset_id":"space.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"place%1:15:05::","call_ids":["senseval2.d000.s029.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"place%1:15:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1064,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00021319999999999998,"input_uncached_usd":0.0000532,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.64293833298143}
{"item_id":"senseval2.d000.s030.t000","gold_sense_keys":["well-known%5:00:00:known:00"],"candidates":[{"index":1,"sense_key":"well-known%5:00:00:known:00","synset_id":"well-known.s.01"},{"index":2,"sense_key":"well-known%5:00:00:familiar:00","synset_id":"long-familiar.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"well-known%5:00:00:known:00","call_ids":["senseval2.d000.s030.t000__v1__a1","senseval2.d000.s030.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"well-known%5:00:00:known:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00039514999999999993,"input_uncached_usd":0.00004955,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.507504041015636}
{"item_id":"senseval2.d000.s030.t002","gold_sense_keys":["involve%2:42:02::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involve%2:42:00::","call_ids":["senseval2.d000.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involve%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00070455,"input_uncached_usd":0.00003255,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.243345041002613}
{"item_id":"senseval2.d000.s030.t003","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046839999999999995,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.213395500031766}
{"item_id":"senseval2.d000.s030.t004","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037974999999999993,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.865192083991133}
{"item_id":"senseval2.d000.s030.t005","gold_sense_keys":["english%3:01:00::"],"candidates":[{"index":1,"sense_key":"english%3:01:00::","synset_id":"english.a.01"},{"index":2,"sense_key":"english%3:01:01::","synset_id":"english.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"english%3:01:00::","call_ids":["senseval2.d000.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"english%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020805,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.135631375014782}
{"item_id":"senseval2.d000.s030.t007","gold_sense_keys":["describe%2:32:00::","describe%2:32:01::"],"candidates":[{"index":1,"sense_key":"describe%2:32:00::","synset_id":"describe.v.01"},{"index":2,"sense_key":"describe%2:32:01::","synset_id":"report.v.01"},{"index":3,"sense_key":"describe%2:35:00::","synset_id":"trace.v.02"},{"index":4,"sense_key":"describe%2:31:00::","synset_id":"identify.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"describe%2:32:01::","call_ids":["senseval2.d000.s030.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"describe%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001626,"input_uncached_usd":0.000028199999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.74033091700403}
{"item_id":"senseval2.d000.s030.t008","gold_sense_keys":["passion%1:09:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"passion%1:12:00::","call_ids":["senseval2.d000.s030.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"passion%1:12:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00065165,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.35570412495872}
{"item_id":"senseval2.d000.s030.t009","gold_sense_keys":["find%2:30:00::","find%2:38:10::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"find%2:31:10::","call_ids":["senseval2.d000.s030.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"find%2:31:10::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006540500000000001,"input_uncached_usd":0.00005885,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.576160375028849}
{"item_id":"senseval2.d000.s030.t010","gold_sense_keys":["satisfaction%1:12:00::","satisfaction%1:26:00::"],"candidates":[{"index":1,"sense_key":"satisfaction%1:12:00::","synset_id":"satisfaction.n.01"},{"index":2,"sense_key":"satisfaction%1:26:00::","synset_id":"gratification.n.01"},{"index":3,"sense_key":"satisfaction%1:21:00::","synset_id":"atonement.n.01"},{"index":4,"sense_key":"satisfaction%1:21:01::","synset_id":"satisfaction.n.04"},{"index":5,"sense_key":"satisfaction%1:04:00::","synset_id":"satisfaction.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"satisfaction%1:26:00::","call_ids":["senseval2.d000.s030.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"satisfaction%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047139999999999997,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.470460334036034}
{"item_id":"senseval2.d000.s030.t011","gold_sense_keys":["mathematical%5:00:00:exact:00"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"mathematical%5:00:00:exact:00","call_ids":["senseval2.d000.s030.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"mathematical%5:00:00:exact:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002917,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.237364917004015}
{"item_id":"senseval2.d000.s030.t012","gold_sense_keys":["completeness%1:26:00::"],"candidates":[{"index":1,"sense_key":"completeness%1:26:00::","synset_id":"completeness.n.01"},{"index":2,"sense_key":"completeness%1:07:00::","synset_id":"completeness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"completeness%1:26:00::","call_ids":["senseval2.d000.s030.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"completeness%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049095,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.097661292005796}
{"item_id":"senseval2.d000.s030.t013","gold_sense_keys":["mechanical%3:00:00::","mechanical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mechanical%3:00:00::","synset_id":"mechanical.a.01"},{"index":2,"sense_key":"mechanical%3:01:01::","synset_id":"mechanical.a.02"},{"index":3,"sense_key":"mechanical%3:01:00::","synset_id":"mechanical.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mechanical%3:00:00::","call_ids":["senseval2.d000.s030.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mechanical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":3600,"reasoning_output_tokens":3584},"cost":{"currency":"USD","total_usd":0.00146595,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.0014399999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":27.473776417027693}
{"item_id":"senseval2.d000.s030.t014","gold_sense_keys":["perfection%1:26:00::"],"candidates":[{"index":1,"sense_key":"perfection%1:26:00::","synset_id":"perfection.n.01"},{"index":2,"sense_key":"perfection%1:09:00::","synset_id":"paragon.n.01"},{"index":3,"sense_key":"perfection%1:04:00::","synset_id":"perfection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"perfection%1:04:00::","call_ids":["senseval2.d000.s030.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"perfection%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054255,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.749793542025145}
{"item_id":"senseval2.d000.s031.t000","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028799999999999995,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.982225458021276}
{"item_id":"senseval2.d000.s031.t001","gold_sense_keys":["add%2:32:01::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"add%2:32:01::","call_ids":["senseval2.d000.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"add%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029509999999999996,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.910429958952591}
{"item_id":"senseval2.d000.s031.t002","gold_sense_keys":["filled%5:00:01:full:00"],"candidates":[{"index":1,"sense_key":"filled%5:00:01:full:00","synset_id":"filled.s.01"},{"index":2,"sense_key":"filled%3:44:00::","synset_id":"filled.a.02"},{"index":3,"sense_key":"filled%5:00:00:occupied:00","synset_id":"filled.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"filled%5:00:01:full:00","call_ids":["senseval2.d000.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"filled%5:00:01:full:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036349999999999995,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.205250541970599}
{"item_id":"senseval2.d000.s031.t003","gold_sense_keys":["solemn%5:00:00:serious:00"],"candidates":[{"index":1,"sense_key":"solemn%5:00:00:serious:00","synset_id":"grave.s.01"},{"index":2,"sense_key":"solemn%5:00:01:serious:00","synset_id":"earnest.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"solemn%5:00:00:serious:00","call_ids":["senseval2.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"solemn%5:00:00:serious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005434,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.663563917041756}
{"item_id":"senseval2.d000.s031.t004","gold_sense_keys":["come%2:30:13::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"come%2:42:04::","call_ids":["senseval2.d000.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"come%2:42:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1249,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00042725,"input_uncached_usd":0.00006245,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.786135124973953}
{"item_id":"senseval2.d000.s031.t006","gold_sense_keys":["ritual%1:04:00::"],"candidates":[{"index":1,"sense_key":"ritual%1:04:00::","synset_id":"ritual.n.01"},{"index":2,"sense_key":"ritual%1:04:01::","synset_id":"ritual.n.02"},{"index":3,"sense_key":"ritual%1:04:02::","synset_id":"ritual.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ritual%1:04:01::","call_ids":["senseval2.d000.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ritual%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048985,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.260661832988262}
{"item_id":"senseval2.d000.s031.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perform%2:36:00::","call_ids":["senseval2.d000.s031.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.681453790981323}
{"item_id":"senseval2.d000.s032.t000","gold_sense_keys":["become%2:42:01::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d000.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031435,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.105389500036836}
{"item_id":"senseval2.d000.s032.t002","gold_sense_keys":["obsession%1:09:00::"],"candidates":[{"index":1,"sense_key":"obsession%1:16:00::","synset_id":"compulsion.n.02"},{"index":2,"sense_key":"obsession%1:09:00::","synset_id":"obsession.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obsession%1:16:00::","call_ids":["senseval2.d000.s032.t002__v1__a1","senseval2.d000.s032.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obsession%1:16:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00057155,"input_uncached_usd":0.00004675,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.878782208950724}
{"item_id":"senseval2.d000.s032.t003","gold_sense_keys":["admit%2:32:00::"],"candidates":[{"index":1,"sense_key":"admit%2:32:00::","synset_id":"admit.v.01"},{"index":2,"sense_key":"admit%2:41:01::","synset_id":"admit.v.02"},{"index":3,"sense_key":"admit%2:41:00::","synset_id":"admit.v.03"},{"index":4,"sense_key":"admit%2:40:00::","synset_id":"accept.v.05"},{"index":5,"sense_key":"admit%2:42:06::","synset_id":"admit.v.05"},{"index":6,"sense_key":"admit%2:42:00::","synset_id":"admit.v.06"},{"index":7,"sense_key":"admit%2:42:04::","synset_id":"accommodate.v.04"},{"index":8,"sense_key":"admit%2:42:03::","synset_id":"admit.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"admit%2:32:00::","call_ids":["senseval2.d000.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"admit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019754999999999998,"input_uncached_usd":0.00003755,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.177808791981079}
{"item_id":"senseval2.d000.s032.t004","gold_sense_keys":["master%1:18:03::"],"candidates":[{"index":1,"sense_key":"master%1:18:00::","synset_id":"maestro.n.01"},{"index":2,"sense_key":"master%1:18:04::","synset_id":"overlord.n.01"},{"index":3,"sense_key":"master%1:18:01::","synset_id":"victor.n.01"},{"index":4,"sense_key":"master%1:18:03::","synset_id":"master.n.04"},{"index":5,"sense_key":"master%1:18:06::","synset_id":"headmaster.n.01"},{"index":6,"sense_key":"master%1:06:00::","synset_id":"master.n.06"},{"index":7,"sense_key":"master%1:18:07::","synset_id":"master.n.07"},{"index":8,"sense_key":"master%1:18:05::","synset_id":"master.n.08"},{"index":9,"sense_key":"master%1:18:02::","synset_id":"master.n.09"},{"index":10,"sense_key":"master%1:06:01::","synset_id":"passkey.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"master%1:18:03::","call_ids":["senseval2.d000.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"master%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037344999999999994,"input_uncached_usd":0.00003425,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2924057909986}
{"item_id":"senseval2.d000.s032.t007","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d000.s032.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1401,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00035805,"input_uncached_usd":0.00007005,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.777039667009376}
{"item_id":"senseval2.d000.s032.t008","gold_sense_keys":["female%3:00:00::"],"candidates":[{"index":1,"sense_key":"female%3:00:00::","synset_id":"female.a.01"},{"index":2,"sense_key":"female%5:00:02:feminine:01","synset_id":"female.s.01"},{"index":3,"sense_key":"female%5:00:01:female:00","synset_id":"female.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"female%5:00:01:female:00","call_ids":["senseval2.d000.s032.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"female%5:00:01:female:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021065,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.609255250019487}
{"item_id":"senseval2.d000.s032.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s032.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033854999999999996,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.21177599998191}
{"item_id":"senseval2.d000.s033.t000","gold_sense_keys":["passion%1:09:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"passion%1:12:00::","call_ids":["senseval2.d000.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"passion%1:12:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005993,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.228490417008288}
{"item_id":"senseval2.d000.s033.t002","gold_sense_keys":["stay%2:42:04::"],"candidates":[{"index":1,"sense_key":"stay%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"stay%2:38:00::","synset_id":"stay.v.02"},{"index":3,"sense_key":"stay%2:42:01::","synset_id":"bide.v.01"},{"index":4,"sense_key":"stay%2:42:02::","synset_id":"stay.v.04"},{"index":5,"sense_key":"stay%2:38:01::","synset_id":"stay.v.05"},{"index":6,"sense_key":"stay%2:30:01::","synset_id":"stay.v.06"},{"index":7,"sense_key":"stay%2:42:04::","synset_id":"persist.v.03"},{"index":8,"sense_key":"stay%2:42:03::","synset_id":"last_out.v.01"},{"index":9,"sense_key":"stay%2:41:00::","synset_id":"stay.v.09"},{"index":10,"sense_key":"stay%2:35:00::","synset_id":"stay.v.10"},{"index":11,"sense_key":"stay%2:34:00::","synset_id":"quell.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stay%2:38:00::","call_ids":["senseval2.d000.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stay%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":3600,"reasoning_output_tokens":3584},"cost":{"currency":"USD","total_usd":0.00148005,"input_uncached_usd":0.00004005,"input_cached_usd":0.0,"output_usd":0.0014399999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":21.097433959017508}
{"item_id":"senseval2.d000.s033.t003","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023495,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.126360291033052}
{"item_id":"senseval2.d000.s033.t004","gold_sense_keys":["however%4:02:00::"],"candidates":[{"index":1,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":2,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":3,"sense_key":"however%4:02:02::","synset_id":"however.r.03"},{"index":4,"sense_key":"however%4:02:01::","synset_id":"however.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"however%4:02:00::","call_ids":["senseval2.d000.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"however%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034155,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.247357707994524}
{"item_id":"senseval2.d000.s034.t000","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005656999999999999,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.815115250006784}
{"item_id":"senseval2.d000.s034.t001","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d000.s034.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003106,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.423437167017255}
{"item_id":"senseval2.d000.s034.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028635,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.650996458018199}
{"item_id":"senseval2.d000.s034.t003","gold_sense_keys":["think_of%2:31:01::"],"candidates":[{"index":1,"sense_key":"think_of%2:31:00::","synset_id":"remember.v.02"},{"index":2,"sense_key":"think_of%2:31:03::","synset_id":"entertain.v.02"},{"index":3,"sense_key":"think_of%2:31:01::","synset_id":"think_of.v.03"},{"index":4,"sense_key":"think_of%2:31:02::","synset_id":"think_of.v.04"},{"index":5,"sense_key":"think_of%2:36:00::","synset_id":"think_up.v.01"},{"index":6,"sense_key":"think_of%2:31:04::","synset_id":"think_of.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"think_of%2:31:01::","call_ids":["senseval2.d000.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"think_of%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019255,"input_uncached_usd":0.00003255,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.169251749990508}
{"item_id":"senseval2.d000.s034.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049185,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.634100041992497}
{"item_id":"senseval2.d000.s034.t006","gold_sense_keys":["bottom%1:15:01::"],"candidates":[{"index":1,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":2,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":3,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"},{"index":4,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":7,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bottom%1:15:01::","call_ids":["senseval2.d000.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bottom%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037059999999999996,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.035788875014987}
{"item_id":"senseval2.d000.s035.t001","gold_sense_keys":["complete%2:30:02::"],"candidates":[{"index":1,"sense_key":"complete%2:30:02::","synset_id":"complete.v.01"},{"index":2,"sense_key":"complete%2:30:00::","synset_id":"complete.v.02"},{"index":3,"sense_key":"complete%2:41:00::","synset_id":"dispatch.v.02"},{"index":4,"sense_key":"complete%2:33:00::","synset_id":"complete.v.04"},{"index":5,"sense_key":"complete%2:32:00::","synset_id":"complete.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complete%2:30:02::","call_ids":["senseval2.d000.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complete%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016005,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.979156040993985}
{"item_id":"senseval2.d000.s035.t002","gold_sense_keys":["work_up%2:29:00::"],"candidates":[{"index":1,"sense_key":"work_up%2:30:00::","synset_id":"build_up.v.02"},{"index":2,"sense_key":"work_up%2:29:00::","synset_id":"work_up.v.02"},{"index":3,"sense_key":"work_up%2:30:06::","synset_id":"build_up.v.04"},{"index":4,"sense_key":"work_up%2:30:03::","synset_id":"work_out.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work_up%2:30:00::","call_ids":["senseval2.d000.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work_up%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036419999999999996,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.158950250013731}
{"item_id":"senseval2.d000.s035.t003","gold_sense_keys":["sweat%1:08:00::"],"candidates":[{"index":1,"sense_key":"sweat%1:08:00::","synset_id":"perspiration.n.01"},{"index":2,"sense_key":"sweat%1:26:01::","synset_id":"fret.n.01"},{"index":3,"sense_key":"sweat%1:19:00::","synset_id":"sweat.n.03"},{"index":4,"sense_key":"sweat%1:04:00::","synset_id":"effort.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sweat%1:04:00::","call_ids":["senseval2.d000.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sweat%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028835,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0146320419735275}
{"item_id":"senseval2.d000.s035.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010619999999999999,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.573509167006705}
{"item_id":"senseval2.d000.s035.t005","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d000.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025844999999999997,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.380360624985769}
{"item_id":"senseval2.d000.s035.t006","gold_sense_keys":["skip%2:38:02::"],"candidates":[{"index":1,"sense_key":"skip%2:31:01::","synset_id":"jump.v.13"},{"index":2,"sense_key":"skip%2:42:00::","synset_id":"cut.v.11"},{"index":3,"sense_key":"skip%2:38:00::","synset_id":"hop.v.01"},{"index":4,"sense_key":"skip%2:38:02::","synset_id":"decamp.v.03"},{"index":5,"sense_key":"skip%2:38:03::","synset_id":"skip.v.05"},{"index":6,"sense_key":"skip%2:35:00::","synset_id":"skim.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"skip%2:38:02::","call_ids":["senseval2.d000.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"skip%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018649999999999998,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.877238707966171}
{"item_id":"senseval2.d000.s035.t007","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval2.d000.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010524999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.315468500019051}
{"item_id":"senseval2.d000.s035.t009","gold_sense_keys":["leave%2:40:06::"],"candidates":[{"index":1,"sense_key":"leave%2:38:01::","synset_id":"leave.v.01"},{"index":2,"sense_key":"leave%2:31:05::","synset_id":"leave.v.02"},{"index":3,"sense_key":"leave%2:42:02::","synset_id":"leave.v.03"},{"index":4,"sense_key":"leave%2:30:02::","synset_id":"leave.v.04"},{"index":5,"sense_key":"leave%2:38:00::","synset_id":"exit.v.01"},{"index":6,"sense_key":"leave%2:42:01::","synset_id":"leave.v.06"},{"index":7,"sense_key":"leave%2:42:00::","synset_id":"leave.v.07"},{"index":8,"sense_key":"leave%2:41:00::","synset_id":"leave.v.08"},{"index":9,"sense_key":"leave%2:40:06::","synset_id":"entrust.v.02"},{"index":10,"sense_key":"leave%2:40:01::","synset_id":"bequeath.v.01"},{"index":11,"sense_key":"leave%2:42:03::","synset_id":"leave.v.11"},{"index":12,"sense_key":"leave%2:30:03::","synset_id":"leave.v.12"},{"index":13,"sense_key":"leave%2:40:02::","synset_id":"impart.v.01"},{"index":14,"sense_key":"leave%2:31:02::","synset_id":"forget.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"leave%2:38:00::","call_ids":["senseval2.d000.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"leave%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1003,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00031255,"input_uncached_usd":0.00005015,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.762790333013982}
{"item_id":"senseval2.d000.s035.t010","gold_sense_keys":["worship%1:04:00::"],"candidates":[{"index":1,"sense_key":"worship%1:04:00::","synset_id":"worship.n.01"},{"index":2,"sense_key":"worship%1:12:01::","synset_id":"worship.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worship%1:04:00::","call_ids":["senseval2.d000.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worship%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":659,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028220000000000003,"input_uncached_usd":0.000018599999999999998,"input_cached_usd":0.0,"output_usd":0.0002636,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.5973209160147235}
{"item_id":"senseval2.d000.s035.t011","gold_sense_keys":["below%4:02:06::"],"candidates":[{"index":1,"sense_key":"below%4:02:01::","synset_id":"below.r.01"},{"index":2,"sense_key":"below%4:02:00::","synset_id":"below.r.02"},{"index":3,"sense_key":"below%4:02:02::","synset_id":"below.r.03"},{"index":4,"sense_key":"below%4:02:06::","synset_id":"downstairs.r.01"},{"index":5,"sense_key":"below%4:02:03::","synset_id":"under.r.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"below%4:02:06::","call_ids":["senseval2.d000.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"below%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00074765,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.036585416004527}
{"item_id":"senseval2.d000.s036.t000","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"well%4:02:07::","call_ids":["senseval2.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"well%4:02:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003583,"input_uncached_usd":0.000044699999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.328789792023599}
{"item_id":"senseval2.d000.s037.t000","gold_sense_keys":["membership%1:14:00::"],"candidates":[{"index":1,"sense_key":"membership%1:14:00::","synset_id":"membership.n.01"},{"index":2,"sense_key":"membership%1:26:00::","synset_id":"membership.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"membership%1:14:00::","call_ids":["senseval2.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"membership%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":2768,"reasoning_output_tokens":2752},"cost":{"currency":"USD","total_usd":0.0011280999999999997,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0011072,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.69153670797823}
{"item_id":"senseval2.d000.s037.t002","gold_sense_keys":["steadily%4:02:01::"],"candidates":[{"index":1,"sense_key":"steadily%4:02:01::","synset_id":"steadily.r.01"},{"index":2,"sense_key":"steadily%4:02:00::","synset_id":"steadily.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"steadily%4:02:01::","call_ids":["senseval2.d000.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"steadily%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005647,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.602866375003941}
{"item_id":"senseval2.d000.s037.t005","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s037.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010534999999999999,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1597999159712344}
{"item_id":"senseval2.d000.s037.t006","gold_sense_keys":["press%2:32:00::"],"candidates":[{"index":1,"sense_key":"press%2:35:00::","synset_id":"press.v.01"},{"index":2,"sense_key":"press%2:32:00::","synset_id":"urge.v.01"},{"index":3,"sense_key":"press%2:42:01::","synset_id":"weigh.v.05"},{"index":4,"sense_key":"press%2:35:01::","synset_id":"press.v.04"},{"index":5,"sense_key":"press%2:35:02::","synset_id":"compress.v.02"},{"index":6,"sense_key":"press%2:38:00::","synset_id":"press.v.06"},{"index":7,"sense_key":"press%2:36:01::","synset_id":"press.v.07"},{"index":8,"sense_key":"press%2:42:00::","synset_id":"press.v.08"},{"index":9,"sense_key":"press%2:41:12::","synset_id":"crusade.v.01"},{"index":10,"sense_key":"press%2:36:00::","synset_id":"press.v.10"},{"index":11,"sense_key":"press%2:35:06::","synset_id":"press.v.11"},{"index":12,"sense_key":"press%2:35:03::","synset_id":"iron.v.01"},{"index":13,"sense_key":"press%2:33:00::","synset_id":"weight-lift.v.01"},{"index":14,"sense_key":"press%2:32:01::","synset_id":"bid.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"press%2:32:00::","call_ids":["senseval2.d000.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"press%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":930,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00023209999999999998,"input_uncached_usd":0.0000465,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.772119375003967}
{"item_id":"senseval2.d000.s037.t007","gold_sense_keys":["equally%4:02:01::"],"candidates":[{"index":1,"sense_key":"equally%4:02:01::","synset_id":"equally.r.01"},{"index":2,"sense_key":"equally%4:02:02::","synset_id":"evenly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equally%4:02:01::","call_ids":["senseval2.d000.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015705,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8052102079964243}
{"item_id":"senseval2.d000.s037.t009","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d000.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031095,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.802074583014473}
{"item_id":"senseval2.d000.s037.t010","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s037.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015869999999999998,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.629365000000689}
{"item_id":"senseval2.d000.s037.t011","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":2,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":3,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":4,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":5,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend%2:42:00::","call_ids":["senseval2.d000.s037.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021574999999999999,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.021413333015516}
{"item_id":"senseval2.d000.s037.t012","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s037.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00012954999999999998,"input_uncached_usd":0.000046349999999999995,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1475145000149496}
{"item_id":"senseval2.d000.s038.t000","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026359999999999996,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4938997499994}
{"item_id":"senseval2.d000.s038.t002","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018175,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9724006250035018}
{"item_id":"senseval2.d000.s038.t003","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval2.d000.s038.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1973,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00038665,"input_uncached_usd":0.00009865,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.421533124987036}
{"item_id":"senseval2.d000.s038.t004","gold_sense_keys":["so%4:02:03::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"so%4:02:03::","call_ids":["senseval2.d000.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"so%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00073875,"input_uncached_usd":0.00004115,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.103557874972466}
{"item_id":"senseval2.d000.s038.t005","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023509999999999997,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3804900830145925}
{"item_id":"senseval2.d000.s038.t006","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":2,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":3,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":4,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":5,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend%2:42:00::","call_ids":["senseval2.d000.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034334999999999997,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.916888292005751}
{"item_id":"senseval2.d000.s038.t007","gold_sense_keys":["service%1:04:01::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"service%1:04:01::","call_ids":["senseval2.d000.s038.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"service%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001548,"input_uncached_usd":0.000046,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6798868749756366}
{"item_id":"senseval2.d000.s038.t008","gold_sense_keys":["sack%2:41:00::"],"candidates":[{"index":1,"sense_key":"sack%2:40:00::","synset_id":"sack.v.01"},{"index":2,"sense_key":"sack%2:41:00::","synset_id":"displace.v.03"},{"index":3,"sense_key":"sack%2:40:01::","synset_id":"net.v.01"},{"index":4,"sense_key":"sack%2:35:00::","synset_id":"sack.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sack%2:40:01::","call_ids":["senseval2.d000.s038.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sack%2:40:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044175,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.717727624985855}
{"item_id":"senseval2.d000.s038.t009","gold_sense_keys":["entire%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"entire%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":2,"sense_key":"entire%5:00:02:whole:00","synset_id":"integral.s.02"},{"index":3,"sense_key":"entire%5:00:00:smooth:02","synset_id":"entire.s.03"},{"index":4,"sense_key":"entire%5:00:00:uncastrated:00","synset_id":"entire.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"entire%5:00:01:whole:00","call_ids":["senseval2.d000.s038.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"entire%5:00:01:whole:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034035,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.11697562498739}
{"item_id":"senseval2.d000.s038.t011","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s038.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028629999999999997,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.391806915984489}
{"item_id":"senseval2.d000.s038.t012","gold_sense_keys":["promptly%4:02:02::"],"candidates":[{"index":1,"sense_key":"promptly%4:02:02::","synset_id":"promptly.r.01"},{"index":2,"sense_key":"promptly%4:02:00::","synset_id":"promptly.r.02"},{"index":3,"sense_key":"promptly%4:02:01::","synset_id":"promptly.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"promptly%4:02:01::","call_ids":["senseval2.d000.s038.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"promptly%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002854,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.762271750019863}
{"item_id":"senseval2.d000.s038.t015","gold_sense_keys":["protest%1:04:00::"],"candidates":[{"index":1,"sense_key":"protest%1:10:00::","synset_id":"protest.n.01"},{"index":2,"sense_key":"protest%1:04:00::","synset_id":"protest.n.02"},{"index":3,"sense_key":"protest%1:10:01::","synset_id":"protest.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"protest%1:04:00::","call_ids":["senseval2.d000.s038.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"protest%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00079865,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.194298332964536}
{"item_id":"senseval2.d000.s039.t000","gold_sense_keys":["club%1:14:00::"],"candidates":[{"index":1,"sense_key":"club%1:14:01::","synset_id":"baseball_club.n.01"},{"index":2,"sense_key":"club%1:14:00::","synset_id":"club.n.02"},{"index":3,"sense_key":"club%1:06:00::","synset_id":"club.n.03"},{"index":4,"sense_key":"club%1:06:02::","synset_id":"clubhouse.n.01"},{"index":5,"sense_key":"club%1:06:01::","synset_id":"golf_club.n.02"},{"index":6,"sense_key":"club%1:06:03::","synset_id":"club.n.06"},{"index":7,"sense_key":"club%1:06:04::","synset_id":"cabaret.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"club%1:14:00::","call_ids":["senseval2.d000.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"club%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002721,"input_uncached_usd":0.0000349,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.084607666998636}
{"item_id":"senseval2.d000.s039.t001","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"treat%2:31:00::","call_ids":["senseval2.d000.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047749999999999995,"input_uncached_usd":0.0000359,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.12600399996154}
{"item_id":"senseval2.d000.s039.t002","gold_sense_keys":["tower%1:06:00::"],"candidates":[{"index":1,"sense_key":"tower%1:06:00::","synset_id":"tower.n.01"},{"index":2,"sense_key":"tower%1:25:00::","synset_id":"column.n.04"},{"index":3,"sense_key":"tower%1:06:01::","synset_id":"tugboat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tower%1:06:00::","call_ids":["senseval2.d000.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tower%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028599999999999996,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.997450416965876}
{"item_id":"senseval2.d000.s039.t003","gold_sense_keys":["sort%1:07:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:09:00::","call_ids":["senseval2.d000.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039119999999999997,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.943521000037435}
{"item_id":"senseval2.d000.s039.t004","gold_sense_keys":["separate%3:00:00::","separate%5:00:00:detached:00"],"candidates":[{"index":1,"sense_key":"separate%3:00:00::","synset_id":"separate.a.01"},{"index":2,"sense_key":"separate%5:00:00:detached:00","synset_id":"freestanding.s.01"},{"index":3,"sense_key":"separate%5:00:02:segregated:00","synset_id":"separate.s.02"},{"index":4,"sense_key":"separate%5:00:02:unconnected:00","synset_id":"disjoined.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"separate%5:00:00:detached:00","call_ids":["senseval2.d000.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"separate%5:00:00:detached:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031415,"input_uncached_usd":0.00002615,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.578512209001929}
{"item_id":"senseval2.d000.s039.t006","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058545,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.780791624973062}
{"item_id":"senseval2.d000.s040.t000","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d000.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004829,"input_uncached_usd":0.0000413,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.68184391700197}
{"item_id":"senseval2.d000.s040.t002","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["senseval2.d000.s040.t002__v1__a1","senseval2.d000.s040.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":1888,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00079955,"input_uncached_usd":0.000044349999999999994,"input_cached_usd":0.0,"output_usd":0.0007551999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.374124707945157}
{"item_id":"senseval2.d000.s040.t003","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d000.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026645,"input_uncached_usd":0.00002965,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.622350500023458}
{"item_id":"senseval2.d000.s040.t004","gold_sense_keys":["congregation%1:14:00::"],"candidates":[{"index":1,"sense_key":"congregation%1:14:00::","synset_id":"congregation.n.01"},{"index":2,"sense_key":"congregation%1:14:01::","synset_id":"congregation.n.02"},{"index":3,"sense_key":"congregation%1:04:00::","synset_id":"congregation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congregation%1:14:00::","call_ids":["senseval2.d000.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congregation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018309999999999998,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.395092124992516}
{"item_id":"senseval2.d000.s041.t000","gold_sense_keys":["still%4:02:01::"],"candidates":[{"index":1,"sense_key":"still%4:02:01::","synset_id":"still.r.01"},{"index":2,"sense_key":"still%4:02:04::","synset_id":"however.r.01"},{"index":3,"sense_key":"still%4:02:00::","synset_id":"even.r.03"},{"index":4,"sense_key":"still%4:02:02::","synset_id":"still.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"still%4:02:01::","call_ids":["senseval2.d000.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"still%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023874999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.742551750037819}
{"item_id":"senseval2.d000.s041.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038845,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.059171375003643}
{"item_id":"senseval2.d000.s041.t003","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044679999999999996,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.97036554099759}
{"item_id":"senseval2.d000.s041.t005","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004541,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.250283375033177}
{"item_id":"senseval2.d000.s042.t000","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003013,"input_uncached_usd":0.0000389,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.297487375035416}
{"item_id":"senseval2.d000.s042.t002","gold_sense_keys":["silent%5:00:00:quiet:01"],"candidates":[{"index":1,"sense_key":"silent%5:00:00:quiet:01","synset_id":"silent.s.01"},{"index":2,"sense_key":"silent%5:00:00:uncommunicative:00","synset_id":"mum.s.01"},{"index":3,"sense_key":"silent%5:00:00:implicit:00","synset_id":"silent.s.03"},{"index":4,"sense_key":"silent%5:00:02:inaudible:00","synset_id":"silent.s.04"},{"index":5,"sense_key":"silent%5:00:00:inaudible:00","synset_id":"silent.s.05"},{"index":6,"sense_key":"silent%5:00:00:inarticulate:00","synset_id":"dumb.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"silent%5:00:00:quiet:01","call_ids":["senseval2.d000.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"silent%5:00:00:quiet:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021634999999999997,"input_uncached_usd":0.000030749999999999995,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.829483124951366}
{"item_id":"senseval2.d000.s042.t003","gold_sense_keys":["follow%2:42:01::"],"candidates":[{"index":1,"sense_key":"follow%2:38:00::","synset_id":"follow.v.01"},{"index":2,"sense_key":"follow%2:42:00::","synset_id":"postdate.v.01"},{"index":3,"sense_key":"follow%2:42:05::","synset_id":"follow.v.03"},{"index":4,"sense_key":"follow%2:38:01::","synset_id":"follow.v.04"},{"index":5,"sense_key":"follow%2:41:00::","synset_id":"comply.v.01"},{"index":6,"sense_key":"follow%2:42:01::","synset_id":"follow.v.06"},{"index":7,"sense_key":"follow%2:30:00::","synset_id":"follow.v.07"},{"index":8,"sense_key":"follow%2:42:02::","synset_id":"follow.v.08"},{"index":9,"sense_key":"follow%2:40:00::","synset_id":"adopt.v.01"},{"index":10,"sense_key":"follow%2:42:03::","synset_id":"follow.v.10"},{"index":11,"sense_key":"follow%2:36:00::","synset_id":"take_after.v.02"},{"index":12,"sense_key":"follow%2:31:00::","synset_id":"trace.v.01"},{"index":13,"sense_key":"follow%2:41:02::","synset_id":"watch.v.02"},{"index":14,"sense_key":"follow%2:41:01::","synset_id":"succeed.v.02"},{"index":15,"sense_key":"follow%2:36:01::","synset_id":"play_along.v.02"},{"index":16,"sense_key":"follow%2:30:01::","synset_id":"keep_up.v.04"},{"index":17,"sense_key":"follow%2:42:04::","synset_id":"come.v.05"},{"index":18,"sense_key":"follow%2:41:10::","synset_id":"follow.v.18"},{"index":19,"sense_key":"follow%2:41:11::","synset_id":"follow.v.19"},{"index":20,"sense_key":"follow%2:41:09::","synset_id":"be.v.08"},{"index":21,"sense_key":"follow%2:39:13::","synset_id":"surveil.v.01"},{"index":22,"sense_key":"follow%2:38:13::","synset_id":"pursue.v.02"},{"index":23,"sense_key":"follow%2:31:14::","synset_id":"follow.v.23"},{"index":24,"sense_key":"follow%2:30:12::","synset_id":"stick_to.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"follow%2:42:01::","call_ids":["senseval2.d000.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"follow%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1397,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00025544999999999995,"input_uncached_usd":0.00006984999999999999,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.584908499964513}
{"item_id":"senseval2.d000.s042.t005","gold_sense_keys":["church%1:04:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"church%1:14:01::","call_ids":["senseval2.d000.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"church%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008251,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.005607583967503}
{"item_id":"senseval2.d000.s043.t000","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013094999999999998,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.829309209017083}
{"item_id":"senseval2.d000.s043.t001","gold_sense_keys":["refuse%2:32:00::"],"candidates":[{"index":1,"sense_key":"refuse%2:32:00::","synset_id":"refuse.v.01"},{"index":2,"sense_key":"refuse%2:40:00::","synset_id":"refuse.v.02"},{"index":3,"sense_key":"refuse%2:42:00::","synset_id":"defy.v.02"},{"index":4,"sense_key":"refuse%2:40:01::","synset_id":"deny.v.04"},{"index":5,"sense_key":"refuse%2:42:04::","synset_id":"resist.v.05"},{"index":6,"sense_key":"refuse%2:41:00::","synset_id":"reject.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"refuse%2:32:00::","call_ids":["senseval2.d000.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"refuse%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029305,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.49160429200856}
{"item_id":"senseval2.d000.s043.t002","gold_sense_keys":["talk_about%2:32:01::"],"candidates":[{"index":1,"sense_key":"talk_about%2:32:00::","synset_id":"discourse.v.01"},{"index":2,"sense_key":"talk_about%2:32:01::","synset_id":"talk_of.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"talk_about%2:32:01::","call_ids":["senseval2.d000.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"talk_about%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064285,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.2163946669898}
{"item_id":"senseval2.d000.s043.t003","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s043.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006624000000000001,"input_uncached_usd":0.000041599999999999995,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.417561540962197}
{"item_id":"senseval2.d000.s044.t000","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007905,"input_uncached_usd":0.00002145,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9795677909860387}
{"item_id":"senseval2.d000.s044.t002","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00095225,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.16843570797937}
{"item_id":"senseval2.d000.s044.t003","gold_sense_keys":["feel%2:31:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"feel%2:31:00::","call_ids":["senseval2.d000.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"feel%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":880,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002808,"input_uncached_usd":0.000044,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1795722499955446}
{"item_id":"senseval2.d000.s044.t004","gold_sense_keys":["fault%1:07:01::"],"candidates":[{"index":1,"sense_key":"fault%1:04:00::","synset_id":"mistake.n.01"},{"index":2,"sense_key":"fault%1:26:00::","synset_id":"defect.n.03"},{"index":3,"sense_key":"fault%1:07:00::","synset_id":"demerit.n.02"},{"index":4,"sense_key":"fault%1:17:00::","synset_id":"fault.n.04"},{"index":5,"sense_key":"fault%1:11:00::","synset_id":"fault.n.05"},{"index":6,"sense_key":"fault%1:07:01::","synset_id":"fault.n.06"},{"index":7,"sense_key":"fault%1:04:01::","synset_id":"fault.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fault%1:26:00::","call_ids":["senseval2.d000.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fault%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029765,"input_uncached_usd":0.000035249999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.831705958000384}
{"item_id":"senseval2.d000.s044.t008","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval2.d000.s044.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010664999999999999,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4875134589965455}
{"item_id":"senseval2.d000.s044.t009","gold_sense_keys":["altar%1:06:01::"],"candidates":[{"index":1,"sense_key":"altar%1:06:01::","synset_id":"altar.n.01"},{"index":2,"sense_key":"altar%1:06:00::","synset_id":"altar.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"altar%1:06:00::","call_ids":["senseval2.d000.s044.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"altar%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.0009735,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.089692165958695}
{"item_id":"senseval2.d000.s045.t000","gold_sense_keys":["so%4:02:10::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"so%4:02:09::","call_ids":["senseval2.d000.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"so%4:02:09::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":3479,"reasoning_output_tokens":3456},"cost":{"currency":"USD","total_usd":0.0014323,"input_uncached_usd":0.0000407,"input_cached_usd":0.0,"output_usd":0.0013916,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":25.422336583957076}
{"item_id":"senseval2.d000.s045.t007","gold_sense_keys":["here%4:02:01::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"here%4:02:01::","call_ids":["senseval2.d000.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"here%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006710999999999999,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.86819483304862}
{"item_id":"senseval2.d000.s045.t008","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval2.d000.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1205,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00029705,"input_uncached_usd":0.000060249999999999994,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.096173167054076}
{"item_id":"senseval2.d000.s045.t009","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:00::","call_ids":["senseval2.d000.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038829999999999995,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.013841542007867}
{"item_id":"senseval2.d000.s045.t010","gold_sense_keys":["above%4:02:01::"],"candidates":[{"index":1,"sense_key":"above%4:02:00::","synset_id":"above.r.01"},{"index":2,"sense_key":"above%4:02:01::","synset_id":"above.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"above%4:02:01::","call_ids":["senseval2.d000.s045.t010__v1__a1","senseval2.d000.s045.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"above%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005415499999999999,"input_uncached_usd":0.00004235,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.274758708023}
{"item_id":"senseval2.d000.s045.t011","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":36,"chosen_sense_key":"make%2:38:10::","call_ids":["senseval2.d000.s045.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":36,"predicted_sense_key":"make%2:38:10::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2289,"cached_input_tokens":0,"output_tokens":2514,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.00112005,"input_uncached_usd":0.00011444999999999999,"input_cached_usd":0.0,"output_usd":0.0010056,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.496348250017036}
{"item_id":"senseval2.d000.s045.t012","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d000.s045.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033505,"input_uncached_usd":0.00002145,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.26746954099508}
{"item_id":"senseval2.d000.s045.t014","gold_sense_keys":["exit%1:04:00::"],"candidates":[{"index":1,"sense_key":"exit%1:06:00::","synset_id":"exit.n.01"},{"index":2,"sense_key":"exit%1:11:00::","synset_id":"passing.n.02"},{"index":3,"sense_key":"exit%1:04:00::","synset_id":"exit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"exit%1:04:00::","call_ids":["senseval2.d000.s045.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"exit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001312,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.763558834034484}
{"item_id":"senseval2.d000.s045.t015","gold_sense_keys":["congregation%1:14:00::"],"candidates":[{"index":1,"sense_key":"congregation%1:14:00::","synset_id":"congregation.n.01"},{"index":2,"sense_key":"congregation%1:14:01::","synset_id":"congregation.n.02"},{"index":3,"sense_key":"congregation%1:04:00::","synset_id":"congregation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congregation%1:14:00::","call_ids":["senseval2.d000.s045.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congregation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018265,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.578597375017125}
{"item_id":"senseval2.d000.s045.t016","gold_sense_keys":["prayer%1:04:00::"],"candidates":[{"index":1,"sense_key":"prayer%1:04:00::","synset_id":"prayer.n.01"},{"index":2,"sense_key":"prayer%1:10:02::","synset_id":"prayer.n.02"},{"index":3,"sense_key":"prayer%1:10:00::","synset_id":"entreaty.n.01"},{"index":4,"sense_key":"prayer%1:10:01::","synset_id":"prayer.n.04"},{"index":5,"sense_key":"prayer%1:18:00::","synset_id":"prayer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prayer%1:04:00::","call_ids":["senseval2.d000.s045.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prayer%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002627,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.874648917000741}
{"item_id":"senseval2.d000.s045.t017","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s045.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005078999999999999,"input_uncached_usd":0.0000407,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.64977399999043}
{"item_id":"senseval2.d000.s046.t000","gold_sense_keys":["admit%2:32:00::"],"candidates":[{"index":1,"sense_key":"admit%2:32:00::","synset_id":"admit.v.01"},{"index":2,"sense_key":"admit%2:41:01::","synset_id":"admit.v.02"},{"index":3,"sense_key":"admit%2:41:00::","synset_id":"admit.v.03"},{"index":4,"sense_key":"admit%2:40:00::","synset_id":"accept.v.05"},{"index":5,"sense_key":"admit%2:42:06::","synset_id":"admit.v.05"},{"index":6,"sense_key":"admit%2:42:00::","synset_id":"admit.v.06"},{"index":7,"sense_key":"admit%2:42:04::","synset_id":"accommodate.v.04"},{"index":8,"sense_key":"admit%2:42:03::","synset_id":"admit.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"admit%2:32:00::","call_ids":["senseval2.d000.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"admit%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001961,"input_uncached_usd":0.000036099999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.708366875012871}
{"item_id":"senseval2.d000.s046.t001","gold_sense_keys":["mixed%5:00:00:heterogeneous:00"],"candidates":[{"index":1,"sense_key":"mixed%5:00:00:heterogeneous:00","synset_id":"assorted.s.01"},{"index":2,"sense_key":"mixed%5:00:00:integrated:00","synset_id":"interracial.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mixed%5:00:00:heterogeneous:00","call_ids":["senseval2.d000.s046.t001__v1__a1","senseval2.d000.s046.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mixed%5:00:00:heterogeneous:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":929,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006480500000000001,"input_uncached_usd":0.000046450000000000004,"input_cached_usd":0.0,"output_usd":0.0006015999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.64512045896845}
{"item_id":"senseval2.d000.s046.t002","gold_sense_keys":["feeling%1:03:00::","feeling%1:09:03::"],"candidates":[{"index":1,"sense_key":"feeling%1:03:00::","synset_id":"feeling.n.01"},{"index":2,"sense_key":"feeling%1:09:03::","synset_id":"impression.n.01"},{"index":3,"sense_key":"feeling%1:26:00::","synset_id":"spirit.n.02"},{"index":4,"sense_key":"feeling%1:09:01::","synset_id":"feeling.n.04"},{"index":5,"sense_key":"feeling%1:09:02::","synset_id":"touch.n.10"},{"index":6,"sense_key":"feeling%1:09:00::","synset_id":"feeling.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feeling%1:03:00::","call_ids":["senseval2.d000.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feeling%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026869999999999997,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.631307583942544}
{"item_id":"senseval2.d000.s046.t003","gold_sense_keys":["issue%1:09:00::","issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:00::","call_ids":["senseval2.d000.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00051085,"input_uncached_usd":0.00004365,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.684865499963053}
{"item_id":"senseval2.d000.s046.t004","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000789,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8969124580034986}
{"item_id":"senseval2.d000.s046.t005","gold_sense_keys":["active%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"active%5:00:00:involved:00","call_ids":["senseval2.d000.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"active%5:00:00:involved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048544999999999995,"input_uncached_usd":0.000043849999999999995,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.9977787919924594}
{"item_id":"senseval2.d000.s046.t006","gold_sense_keys":["bell_ringer%1:18:01::"],"candidates":[{"index":1,"sense_key":"bell_ringer%1:18:01::","synset_id":"toller.n.01"},{"index":2,"sense_key":"bell_ringer%1:18:00::","synset_id":"bell_ringer.n.02"},{"index":3,"sense_key":"bell_ringer%1:04:00::","synset_id":"bell_ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell_ringer%1:18:01::","call_ids":["senseval2.d000.s046.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell_ringer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015715,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.306712625024375}
{"item_id":"senseval2.d000.s047.t000","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval2.d000.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007042999999999999,"input_uncached_usd":0.0000323,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.529385250003543}
{"item_id":"senseval2.d000.s047.t001","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell%1:11:00::","call_ids":["senseval2.d000.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027479999999999996,"input_uncached_usd":0.000037999999999999995,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.230848791950848}
{"item_id":"senseval2.d000.s047.t002","gold_sense_keys":["draw%2:35:06::"],"candidates":[{"index":1,"sense_key":"draw%2:35:03::","synset_id":"pull.v.01"},{"index":2,"sense_key":"draw%2:40:00::","synset_id":"reap.v.02"},{"index":3,"sense_key":"draw%2:35:01::","synset_id":"trace.v.02"},{"index":4,"sense_key":"draw%2:31:00::","synset_id":"draw.v.04"},{"index":5,"sense_key":"draw%2:38:01::","synset_id":"draw.v.05"},{"index":6,"sense_key":"draw%2:36:00::","synset_id":"draw.v.06"},{"index":7,"sense_key":"draw%2:38:00::","synset_id":"draw.v.07"},{"index":8,"sense_key":"draw%2:32:00::","synset_id":"describe.v.01"},{"index":9,"sense_key":"draw%2:31:13::","synset_id":"draw.v.09"},{"index":10,"sense_key":"draw%2:37:01::","synset_id":"draw.v.10"},{"index":11,"sense_key":"draw%2:34:01::","synset_id":"puff.v.02"},{"index":12,"sense_key":"draw%2:38:02::","synset_id":"draw.v.12"},{"index":13,"sense_key":"draw%2:40:01::","synset_id":"withdraw.v.09"},{"index":14,"sense_key":"draw%2:33:01::","synset_id":"draw.v.14"},{"index":15,"sense_key":"draw%2:36:01::","synset_id":"draw.v.15"},{"index":16,"sense_key":"draw%2:30:14::","synset_id":"draw.v.16"},{"index":17,"sense_key":"draw%2:38:07::","synset_id":"draw.v.17"},{"index":18,"sense_key":"draw%2:36:07::","synset_id":"draw.v.18"},{"index":19,"sense_key":"draw%2:36:02::","synset_id":"draw.v.19"},{"index":20,"sense_key":"draw%2:35:02::","synset_id":"draw.v.20"},{"index":21,"sense_key":"draw%2:42:00::","synset_id":"draw.v.21"},{"index":22,"sense_key":"draw%2:42:02::","synset_id":"draw.v.22"},{"index":23,"sense_key":"draw%2:41:13::","synset_id":"draw.v.23"},{"index":24,"sense_key":"draw%2:38:11::","synset_id":"pull.v.09"},{"index":25,"sense_key":"draw%2:35:04::","synset_id":"absorb.v.04"},{"index":26,"sense_key":"draw%2:35:06::","synset_id":"attract.v.01"},{"index":27,"sense_key":"draw%2:35:14::","synset_id":"string.v.01"},{"index":28,"sense_key":"draw%2:35:09::","synset_id":"pull_back.v.04"},{"index":29,"sense_key":"draw%2:35:13::","synset_id":"guide.v.05"},{"index":30,"sense_key":"draw%2:33:00::","synset_id":"tie.v.02"},{"index":31,"sense_key":"draw%2:30:00::","synset_id":"draw.v.31"},{"index":32,"sense_key":"draw%2:30:08::","synset_id":"draw.v.32"},{"index":33,"sense_key":"draw%2:30:15::","synset_id":"draw.v.33"},{"index":34,"sense_key":"draw%2:30:05::","synset_id":"disembowel.v.01"},{"index":35,"sense_key":"draw%2:30:04::","synset_id":"draw.v.35"},{"index":36,"sense_key":"draw%2:29:00::","synset_id":"draw.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":26,"chosen_sense_key":"draw%2:35:06::","call_ids":["senseval2.d000.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":26,"predicted_sense_key":"draw%2:35:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1865,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00045845,"input_uncached_usd":0.00009324999999999999,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.754188457969576}
{"item_id":"senseval2.d000.s047.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d000.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005183000000000001,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.244227208022494}
{"item_id":"senseval2.d000.s047.t004","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00084975,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.08679850003682}
{"item_id":"senseval2.d000.s047.t005","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s047.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003543,"input_uncached_usd":0.0000407,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.99150683399057}
{"item_id":"senseval2.d000.s048.t000","gold_sense_keys":["live%2:42:06::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"live%2:42:07::","call_ids":["senseval2.d000.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"live%2:42:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.000681,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.269662333012093}
{"item_id":"senseval2.d000.s048.t001","gold_sense_keys":["hope%1:12:01::"],"candidates":[{"index":1,"sense_key":"hope%1:12:01::","synset_id":"hope.n.01"},{"index":2,"sense_key":"hope%1:12:00::","synset_id":"hope.n.02"},{"index":3,"sense_key":"hope%1:09:00::","synset_id":"promise.n.02"},{"index":4,"sense_key":"hope%1:18:00::","synset_id":"hope.n.04"},{"index":5,"sense_key":"hope%1:18:01::","synset_id":"hope.n.05"},{"index":6,"sense_key":"hope%1:07:00::","synset_id":"hope.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hope%1:12:00::","call_ids":["senseval2.d000.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hope%1:12:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029065,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.845804499986116}
{"item_id":"senseval2.d000.s048.t002","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038915,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.547350875043776}
{"item_id":"senseval2.d000.s048.t003","gold_sense_keys":["draw%2:30:14::"],"candidates":[{"index":1,"sense_key":"draw%2:35:03::","synset_id":"pull.v.01"},{"index":2,"sense_key":"draw%2:40:00::","synset_id":"reap.v.02"},{"index":3,"sense_key":"draw%2:35:01::","synset_id":"trace.v.02"},{"index":4,"sense_key":"draw%2:31:00::","synset_id":"draw.v.04"},{"index":5,"sense_key":"draw%2:38:01::","synset_id":"draw.v.05"},{"index":6,"sense_key":"draw%2:36:00::","synset_id":"draw.v.06"},{"index":7,"sense_key":"draw%2:38:00::","synset_id":"draw.v.07"},{"index":8,"sense_key":"draw%2:32:00::","synset_id":"describe.v.01"},{"index":9,"sense_key":"draw%2:31:13::","synset_id":"draw.v.09"},{"index":10,"sense_key":"draw%2:37:01::","synset_id":"draw.v.10"},{"index":11,"sense_key":"draw%2:34:01::","synset_id":"puff.v.02"},{"index":12,"sense_key":"draw%2:38:02::","synset_id":"draw.v.12"},{"index":13,"sense_key":"draw%2:40:01::","synset_id":"withdraw.v.09"},{"index":14,"sense_key":"draw%2:33:01::","synset_id":"draw.v.14"},{"index":15,"sense_key":"draw%2:36:01::","synset_id":"draw.v.15"},{"index":16,"sense_key":"draw%2:30:14::","synset_id":"draw.v.16"},{"index":17,"sense_key":"draw%2:38:07::","synset_id":"draw.v.17"},{"index":18,"sense_key":"draw%2:36:07::","synset_id":"draw.v.18"},{"index":19,"sense_key":"draw%2:36:02::","synset_id":"draw.v.19"},{"index":20,"sense_key":"draw%2:35:02::","synset_id":"draw.v.20"},{"index":21,"sense_key":"draw%2:42:00::","synset_id":"draw.v.21"},{"index":22,"sense_key":"draw%2:42:02::","synset_id":"draw.v.22"},{"index":23,"sense_key":"draw%2:41:13::","synset_id":"draw.v.23"},{"index":24,"sense_key":"draw%2:38:11::","synset_id":"pull.v.09"},{"index":25,"sense_key":"draw%2:35:04::","synset_id":"absorb.v.04"},{"index":26,"sense_key":"draw%2:35:06::","synset_id":"attract.v.01"},{"index":27,"sense_key":"draw%2:35:14::","synset_id":"string.v.01"},{"index":28,"sense_key":"draw%2:35:09::","synset_id":"pull_back.v.04"},{"index":29,"sense_key":"draw%2:35:13::","synset_id":"guide.v.05"},{"index":30,"sense_key":"draw%2:33:00::","synset_id":"tie.v.02"},{"index":31,"sense_key":"draw%2:30:00::","synset_id":"draw.v.31"},{"index":32,"sense_key":"draw%2:30:08::","synset_id":"draw.v.32"},{"index":33,"sense_key":"draw%2:30:15::","synset_id":"draw.v.33"},{"index":34,"sense_key":"draw%2:30:05::","synset_id":"disembowel.v.01"},{"index":35,"sense_key":"draw%2:30:04::","synset_id":"draw.v.35"},{"index":36,"sense_key":"draw%2:29:00::","synset_id":"draw.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":26,"chosen_sense_key":"draw%2:35:06::","call_ids":["senseval2.d000.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":26,"predicted_sense_key":"draw%2:35:06::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1875,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00058655,"input_uncached_usd":0.00009375,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.1196187500027}
{"item_id":"senseval2.d000.s048.t004","gold_sense_keys":["full%3:00:00::"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"full%5:00:00:complete:00","call_ids":["senseval2.d000.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"full%5:00:00:complete:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044934999999999994,"input_uncached_usd":0.00003335,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.718963792023715}
{"item_id":"senseval2.d000.s048.t005","gold_sense_keys":["life%1:26:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"life%1:26:02::","call_ids":["senseval2.d000.s048.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"life%1:26:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":908,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004102,"input_uncached_usd":0.0000454,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.065103625005577}
{"item_id":"senseval2.d000.s049.t000","gold_sense_keys":["sort%1:07:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:09:00::","call_ids":["senseval2.d000.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031515000000000004,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.212688207975589}
{"item_id":"senseval2.d000.s049.t001","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"},{"index":2,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parliament%1:14:00::","call_ids":["senseval2.d000.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007451000000000001,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.365415959036909}
{"item_id":"senseval2.d000.s049.t002","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020915,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.348041124991141}
{"item_id":"senseval2.d000.s049.t003","gold_sense_keys":["aim%2:31:00::","aim%2:31:01::"],"candidates":[{"index":1,"sense_key":"aim%2:33:00::","synset_id":"aim.v.01"},{"index":2,"sense_key":"aim%2:31:01::","synset_id":"aim.v.02"},{"index":3,"sense_key":"aim%2:32:09::","synset_id":"drive.v.11"},{"index":4,"sense_key":"aim%2:31:02::","synset_id":"calculate.v.05"},{"index":5,"sense_key":"aim%2:33:10::","synset_id":"target.v.01"},{"index":6,"sense_key":"aim%2:32:00::","synset_id":"aim.v.06"},{"index":7,"sense_key":"aim%2:31:00::","synset_id":"draw_a_bead_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"aim%2:31:01::","call_ids":["senseval2.d000.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"aim%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034669999999999997,"input_uncached_usd":0.0000331,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.391513500013389}
{"item_id":"senseval2.d000.s049.t004","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["senseval2.d000.s049.t004__v1__a1","senseval2.d000.s049.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00044314999999999996,"input_uncached_usd":0.00004635,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.581166833930183}
{"item_id":"senseval2.d000.s049.t006","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010559999999999999,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1952964999945834}
{"item_id":"senseval2.d000.s049.t007","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058585,"input_uncached_usd":0.00004185,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.406911541998852}
{"item_id":"senseval2.d000.s049.t008","gold_sense_keys":["president%1:18:03::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"president%1:18:03::","call_ids":["senseval2.d000.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"president%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":1106,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047204999999999995,"input_uncached_usd":0.00002965,"input_cached_usd":0.0,"output_usd":0.00044239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.733391666959506}
{"item_id":"senseval2.d000.s050.t000","gold_sense_keys":["hope%2:31:00::"],"candidates":[{"index":1,"sense_key":"hope%2:37:01::","synset_id":"hope.v.01"},{"index":2,"sense_key":"hope%2:37:00::","synset_id":"hope.v.02"},{"index":3,"sense_key":"hope%2:31:00::","synset_id":"hope.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hope%2:31:00::","call_ids":["senseval2.d000.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hope%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002095,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.691486124997027}
{"item_id":"senseval2.d000.s050.t001","gold_sense_keys":["speak%2:32:01::","speak%2:32:03::"],"candidates":[{"index":1,"sense_key":"speak%2:32:00::","synset_id":"talk.v.02"},{"index":2,"sense_key":"speak%2:32:01::","synset_id":"talk.v.01"},{"index":3,"sense_key":"speak%2:32:02::","synset_id":"speak.v.03"},{"index":4,"sense_key":"speak%2:32:03::","synset_id":"address.v.02"},{"index":5,"sense_key":"speak%2:39:00::","synset_id":"speak.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"speak%2:32:03::","call_ids":["senseval2.d000.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"speak%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028914999999999995,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.8333900409634225}
{"item_id":"senseval2.d000.s050.t002","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d000.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041175,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.527021958027035}
{"item_id":"senseval2.d000.s050.t004","gold_sense_keys":["college%1:14:01::"],"candidates":[{"index":1,"sense_key":"college%1:14:00::","synset_id":"college.n.01"},{"index":2,"sense_key":"college%1:14:01::","synset_id":"college.n.02"},{"index":3,"sense_key":"college%1:06:00::","synset_id":"college.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"college%1:14:01::","call_ids":["senseval2.d000.s050.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"college%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023304999999999997,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.305805834010243}
{"item_id":"senseval2.d000.s050.t005","gold_sense_keys":["joy%1:09:00::"],"candidates":[{"index":1,"sense_key":"joy%1:12:00::","synset_id":"joy.n.01"},{"index":2,"sense_key":"joy%1:09:00::","synset_id":"joy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"joy%1:09:00::","call_ids":["senseval2.d000.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"joy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036034999999999995,"input_uncached_usd":0.00002115,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.371123416989576}
{"item_id":"senseval2.d000.s050.t006","gold_sense_keys":["bell_ringing%1:04:00::"],"candidates":[{"index":1,"sense_key":"bell_ringing%1:11:00::","synset_id":"bell_ringing.n.01"},{"index":2,"sense_key":"bell_ringing%1:10:00::","synset_id":"electioneering.n.01"},{"index":3,"sense_key":"bell_ringing%1:04:00::","synset_id":"bell_ringing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell_ringing%1:04:00::","call_ids":["senseval2.d000.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell_ringing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015719999999999997,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9386028749868274}
{"item_id":"senseval2.d000.s050.t007","gold_sense_keys":["shortly%4:02:00::"],"candidates":[{"index":1,"sense_key":"shortly%4:02:01::","synset_id":"shortly.r.01"},{"index":2,"sense_key":"shortly%4:02:00::","synset_id":"soon.r.01"},{"index":3,"sense_key":"shortly%4:02:02::","synset_id":"curtly.r.01"},{"index":4,"sense_key":"shortly%4:02:04::","synset_id":"concisely.r.01"},{"index":5,"sense_key":"shortly%4:02:03::","synset_id":"shortly.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shortly%4:02:00::","call_ids":["senseval2.d000.s050.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shortly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018899999999999999,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.96080316696316}
{"item_id":"senseval2.d000.s050.t008","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["senseval2.d000.s050.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015889999999999999,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.168375083012506}
{"item_id":"senseval2.d000.s050.t010","gold_sense_keys":["vicar%1:18:01::"],"candidates":[{"index":1,"sense_key":"vicar%1:18:00::","synset_id":"vicar.n.01"},{"index":2,"sense_key":"vicar%1:18:02::","synset_id":"vicar.n.02"},{"index":3,"sense_key":"vicar%1:18:01::","synset_id":"vicar.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vicar%1:18:01::","call_ids":["senseval2.d000.s050.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vicar%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015644999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.340069375000894}
{"item_id":"senseval2.d000.s050.t011","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["senseval2.d000.s050.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036894999999999994,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.703791000007186}
{"item_id":"senseval2.d000.s050.t012","gold_sense_keys":["entitle%2:32:00::"],"candidates":[{"index":1,"sense_key":"entitle%2:41:00::","synset_id":"entitle.v.01"},{"index":2,"sense_key":"entitle%2:32:00::","synset_id":"entitle.v.02"},{"index":3,"sense_key":"entitle%2:41:01::","synset_id":"ennoble.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"entitle%2:32:00::","call_ids":["senseval2.d000.s050.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"entitle%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002849,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.109508042049129}
{"item_id":"senseval2.d000.s050.t013","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s050.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":914,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004043,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.0003656,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.787987749965396}
{"item_id":"senseval2.d000.s050.t014","gold_sense_keys":["care%1:04:00::","care%1:04:01::","care%1:04:02::"],"candidates":[{"index":1,"sense_key":"care%1:04:01::","synset_id":"care.n.01"},{"index":2,"sense_key":"care%1:09:00::","synset_id":"caution.n.03"},{"index":3,"sense_key":"care%1:12:01::","synset_id":"concern.n.02"},{"index":4,"sense_key":"care%1:12:00::","synset_id":"care.n.04"},{"index":5,"sense_key":"care%1:04:02::","synset_id":"care.n.05"},{"index":6,"sense_key":"care%1:04:00::","synset_id":"care.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"care%1:04:02::","call_ids":["senseval2.d000.s050.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"care%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001651,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7914597500348464}
{"item_id":"senseval2.d000.s051.t000","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032825,"input_uncached_usd":0.000040249999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.728304625023156}
{"item_id":"senseval2.d000.s051.t002","gold_sense_keys":["recognize%2:31:01::"],"candidates":[{"index":1,"sense_key":"recognize%2:31:02::","synset_id":"acknowledge.v.06"},{"index":2,"sense_key":"recognize%2:31:01::","synset_id":"recognize.v.02"},{"index":3,"sense_key":"recognize%2:39:01::","synset_id":"spot.v.02"},{"index":4,"sense_key":"recognize%2:31:00::","synset_id":"recognize.v.04"},{"index":5,"sense_key":"recognize%2:41:00::","synset_id":"accredit.v.01"},{"index":6,"sense_key":"recognize%2:32:01::","synset_id":"greet.v.01"},{"index":7,"sense_key":"recognize%2:32:00::","synset_id":"acknowledge.v.04"},{"index":8,"sense_key":"recognize%2:42:00::","synset_id":"recognize.v.08"},{"index":9,"sense_key":"recognize%2:41:01::","synset_id":"recognize.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recognize%2:31:01::","call_ids":["senseval2.d000.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recognize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014504999999999997,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.798947292030789}
{"item_id":"senseval2.d000.s051.t003","gold_sense_keys":["no%4:02:01::"],"candidates":[{"index":1,"sense_key":"no%4:02:02::","synset_id":"no.r.01"},{"index":2,"sense_key":"no%4:02:01::","synset_id":"no.r.02"},{"index":3,"sense_key":"no%4:02:00::","synset_id":"no.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"no%4:02:01::","call_ids":["senseval2.d000.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"no%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038675,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.761442708026152}
{"item_id":"senseval2.d000.s051.t005","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval2.d000.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019009999999999999,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.156419000006281}
{"item_id":"senseval2.d000.s051.t006","gold_sense_keys":["priority%1:26:00::"],"candidates":[{"index":1,"sense_key":"priority%1:26:00::","synset_id":"precedence.n.01"},{"index":2,"sense_key":"priority%1:07:00::","synset_id":"priority.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"priority%1:26:00::","call_ids":["senseval2.d000.s051.t006__v1__a1","senseval2.d000.s051.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"priority%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":875,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006197500000000001,"input_uncached_usd":0.00004374999999999999,"input_cached_usd":0.0,"output_usd":0.000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.47566304198699}
{"item_id":"senseval2.d000.s051.t007","gold_sense_keys":["church%1:14:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"church%1:14:00::","call_ids":["senseval2.d000.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"church%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0007982000000000001,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.448009542014915}
{"item_id":"senseval2.d000.s051.t008","gold_sense_keys":["life%1:26:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"life%1:26:01::","call_ids":["senseval2.d000.s051.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"life%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004094,"input_uncached_usd":0.0000446,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.868110625015106}
{"item_id":"senseval2.d000.s051.t009","gold_sense_keys":["experience%1:09:00::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"experience%1:09:00::","call_ids":["senseval2.d000.s051.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"experience%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005665499999999999,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.68671587499557}
{"item_id":"senseval2.d000.s052.t002","gold_sense_keys":["attack%2:30:00::"],"candidates":[{"index":1,"sense_key":"attack%2:33:00::","synset_id":"attack.v.01"},{"index":2,"sense_key":"attack%2:32:00::","synset_id":"attack.v.02"},{"index":3,"sense_key":"attack%2:33:01::","synset_id":"attack.v.03"},{"index":4,"sense_key":"attack%2:33:02::","synset_id":"assail.v.01"},{"index":5,"sense_key":"attack%2:30:00::","synset_id":"attack.v.05"},{"index":6,"sense_key":"attack%2:29:00::","synset_id":"attack.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"attack%2:30:00::","call_ids":["senseval2.d000.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"attack%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037035,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.054475874989294}
{"item_id":"senseval2.d000.s052.t004","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["senseval2.d000.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023344999999999998,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.998477875022218}
{"item_id":"senseval2.d000.s052.t006","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s052.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018309999999999998,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.513562292035203}
{"item_id":"senseval2.d000.s053.t000","gold_sense_keys":["survey%1:04:02::"],"candidates":[{"index":1,"sense_key":"survey%1:04:02::","synset_id":"survey.n.01"},{"index":2,"sense_key":"survey%1:10:00::","synset_id":"sketch.n.03"},{"index":3,"sense_key":"survey%1:04:01::","synset_id":"view.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survey%1:04:02::","call_ids":["senseval2.d000.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survey%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004891499999999999,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.587861499981955}
{"item_id":"senseval2.d000.s053.t001","gold_sense_keys":["say%2:32:01::","say%2:42:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":1873,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007900499999999999,"input_uncached_usd":0.00004085,"input_cached_usd":0.0,"output_usd":0.0007492,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.744458750006743}
{"item_id":"senseval2.d000.s053.t003","gold_sense_keys":["bell_ringer%1:18:01::"],"candidates":[{"index":1,"sense_key":"bell_ringer%1:18:01::","synset_id":"toller.n.01"},{"index":2,"sense_key":"bell_ringer%1:18:00::","synset_id":"bell_ringer.n.02"},{"index":3,"sense_key":"bell_ringer%1:04:00::","synset_id":"bell_ringer.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell_ringer%1:18:01::","call_ids":["senseval2.d000.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell_ringer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003365,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.103743999963626}
{"item_id":"senseval2.d000.s053.t005","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s053.t005__v1__a1","senseval2.d000.s053.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":1313,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056865,"input_uncached_usd":0.00004345,"input_cached_usd":0.0,"output_usd":0.0005252,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.431990999961272}
{"item_id":"senseval2.d000.s053.t006","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s053.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":2513,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.0010359,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.0010052,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.297035791969392}
{"item_id":"senseval2.d000.s053.t007","gold_sense_keys":["still%4:02:01::"],"candidates":[{"index":1,"sense_key":"still%4:02:01::","synset_id":"still.r.01"},{"index":2,"sense_key":"still%4:02:04::","synset_id":"however.r.01"},{"index":3,"sense_key":"still%4:02:00::","synset_id":"even.r.03"},{"index":4,"sense_key":"still%4:02:02::","synset_id":"still.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"still%4:02:01::","call_ids":["senseval2.d000.s053.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"still%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002903,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.262214832997415}
{"item_id":"senseval2.d000.s054.t001","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s054.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023535,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.076304500049446}
{"item_id":"senseval2.d000.s054.t002","gold_sense_keys":["always%4:02:04::"],"candidates":[{"index":1,"sense_key":"always%4:02:00::","synset_id":"always.r.01"},{"index":2,"sense_key":"always%4:02:04::","synset_id":"constantly.r.01"},{"index":3,"sense_key":"always%4:02:03::","synset_id":"constantly.r.02"},{"index":4,"sense_key":"always%4:02:02::","synset_id":"always.r.04"},{"index":5,"sense_key":"always%4:02:01::","synset_id":"always.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"always%4:02:04::","call_ids":["senseval2.d000.s054.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"always%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":2128,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00087895,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.0008512,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.130609542015009}
{"item_id":"senseval2.d000.s054.t003","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval2.d000.s054.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002712,"input_uncached_usd":0.000034399999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.929326708021108}
{"item_id":"senseval2.d000.s054.t004","gold_sense_keys":["bell%1:06:00::"],"candidates":[{"index":1,"sense_key":"bell%1:06:00::","synset_id":"bell.n.01"},{"index":2,"sense_key":"bell%1:06:02::","synset_id":"doorbell.n.01"},{"index":3,"sense_key":"bell%1:11:00::","synset_id":"bell.n.03"},{"index":4,"sense_key":"bell%1:28:00::","synset_id":"bell.n.04"},{"index":5,"sense_key":"bell%1:25:00::","synset_id":"bell.n.05"},{"index":6,"sense_key":"bell%1:18:02::","synset_id":"bell.n.06"},{"index":7,"sense_key":"bell%1:18:01::","synset_id":"bell.n.07"},{"index":8,"sense_key":"bell%1:18:00::","synset_id":"bell.n.08"},{"index":9,"sense_key":"bell%1:06:04::","synset_id":"chime.n.01"},{"index":10,"sense_key":"bell%1:06:03::","synset_id":"bell.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bell%1:06:00::","call_ids":["senseval2.d000.s054.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bell%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042865,"input_uncached_usd":0.00003825,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.9954553749994375}
{"item_id":"senseval2.d000.s054.t005","gold_sense_keys":["need%2:34:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d000.s054.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"need%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023434999999999998,"input_uncached_usd":0.00002315,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.952353874978144}
{"item_id":"senseval2.d000.s054.t006","gold_sense_keys":["ring%2:39:01::"],"candidates":[{"index":1,"sense_key":"ring%2:39:00::","synset_id":"ring.v.01"},{"index":2,"sense_key":"ring%2:39:04::","synset_id":"resound.v.01"},{"index":3,"sense_key":"ring%2:39:01::","synset_id":"ring.v.03"},{"index":4,"sense_key":"ring%2:32:00::","synset_id":"call.v.03"},{"index":5,"sense_key":"ring%2:35:03::","synset_id":"surround.v.01"},{"index":6,"sense_key":"ring%2:35:00::","synset_id":"ring.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ring%2:39:01::","call_ids":["senseval2.d000.s054.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ring%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029335,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5122521669836715}
{"item_id":"senseval2.d000.s054.t007","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"small%5:00:00:limited:00","call_ids":["senseval2.d000.s054.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"small%5:00:00:limited:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003523,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.062215457961429}
{"item_id":"senseval2.d000.s055.t000","gold_sense_keys":["council%1:14:01::"],"candidates":[{"index":1,"sense_key":"council%1:14:01::","synset_id":"council.n.01"},{"index":2,"sense_key":"council%1:14:02::","synset_id":"council.n.02"},{"index":3,"sense_key":"council%1:14:00::","synset_id":"council.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"council%1:14:01::","call_ids":["senseval2.d000.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"council%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005146500000000001,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.004494291963056}
{"item_id":"senseval2.d000.s055.t001","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"program%1:09:00::","call_ids":["senseval2.d000.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"program%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032405,"input_uncached_usd":0.000036049999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.395178625010885}
{"item_id":"senseval2.d000.s055.t002","gold_sense_keys":["attract%2:35:00::"],"candidates":[{"index":1,"sense_key":"attract%2:35:00::","synset_id":"attract.v.01"},{"index":2,"sense_key":"attract%2:37:00::","synset_id":"attract.v.02"},{"index":3,"sense_key":"attract%2:35:01::","synset_id":"attract.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attract%2:37:00::","call_ids":["senseval2.d000.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attract%2:37:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003124,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9793086660210975}
{"item_id":"senseval2.d000.s055.t003","gold_sense_keys":["train%2:31:01::","train%2:32:00::","train%2:41:00::"],"candidates":[{"index":1,"sense_key":"train%2:31:01::","synset_id":"train.v.01"},{"index":2,"sense_key":"train%2:31:00::","synset_id":"train.v.02"},{"index":3,"sense_key":"train%2:41:02::","synset_id":"discipline.v.01"},{"index":4,"sense_key":"train%2:41:00::","synset_id":"prepare.v.05"},{"index":5,"sense_key":"train%2:41:01::","synset_id":"educate.v.03"},{"index":6,"sense_key":"train%2:33:00::","synset_id":"aim.v.01"},{"index":7,"sense_key":"train%2:32:00::","synset_id":"coach.v.01"},{"index":8,"sense_key":"train%2:29:00::","synset_id":"train.v.08"},{"index":9,"sense_key":"train%2:41:03::","synset_id":"train.v.09"},{"index":10,"sense_key":"train%2:38:00::","synset_id":"train.v.10"},{"index":11,"sense_key":"train%2:35:04::","synset_id":"trail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"train%2:32:00::","call_ids":["senseval2.d000.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"train%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048484999999999994,"input_uncached_usd":0.00004325,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.513999458984472}
{"item_id":"senseval2.d000.s055.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023495,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.563649499963503}
{"item_id":"senseval2.d000.s055.t005","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d000.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039525,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.184246416960377}
{"item_id":"senseval2.d000.s055.t008","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d000.s055.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006102000000000001,"input_uncached_usd":0.0000406,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.705666166963056}
{"item_id":"senseval2.d000.s056.t000","gold_sense_keys":["right%4:02:04::"],"candidates":[{"index":1,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":2,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":3,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":4,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":5,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":6,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":7,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":8,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":9,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":10,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"right%4:02:08::","call_ids":["senseval2.d000.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"right%4:02:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039959999999999996,"input_uncached_usd":0.0000348,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1264317920431495}
{"item_id":"senseval2.d000.s056.t001","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d000.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001915,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9424805829767138}
{"item_id":"senseval2.d000.s056.t002","gold_sense_keys":["lucky%3:00:00::"],"candidates":[{"index":1,"sense_key":"lucky%5:00:00:fortunate:00","synset_id":"lucky.s.01"},{"index":2,"sense_key":"lucky%3:00:00::","synset_id":"lucky.a.02"},{"index":3,"sense_key":"lucky%5:00:00:propitious:00","synset_id":"golden.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lucky%5:00:00:fortunate:00","call_ids":["senseval2.d000.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lucky%5:00:00:fortunate:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006431999999999999,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.277541833987925}
{"item_id":"senseval2.d000.s056.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s056.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003139,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.3859587499755435}
{"item_id":"senseval2.d000.s056.t005","gold_sense_keys":["new%3:00:00::","new%5:00:00:inexperienced:00","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"new%5:00:00:inexperienced:00","call_ids":["senseval2.d000.s056.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"new%5:00:00:inexperienced:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001747,"input_uncached_usd":0.0000403,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.915979291952681}
{"item_id":"senseval2.d000.s056.t006","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s056.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013174999999999998,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.407761415990535}
{"item_id":"senseval2.d000.s056.t007","gold_sense_keys":["add%2:32:01::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"add%2:32:01::","call_ids":["senseval2.d000.s056.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"add%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003436,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.64268670801539}
{"item_id":"senseval2.d000.s057.t000","gold_sense_keys":["bright%5:00:00:auspicious:00"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"bright%5:00:00:auspicious:00","call_ids":["senseval2.d000.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"bright%5:00:00:auspicious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022475,"input_uncached_usd":0.000039149999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.209044791990891}
{"item_id":"senseval2.d000.s057.t001","gold_sense_keys":["sign%1:10:05::"],"candidates":[{"index":1,"sense_key":"sign%1:10:05::","synset_id":"sign.n.01"},{"index":2,"sense_key":"sign%1:10:00::","synset_id":"sign.n.02"},{"index":3,"sense_key":"sign%1:10:03::","synset_id":"signal.n.01"},{"index":4,"sense_key":"sign%1:06:01::","synset_id":"signboard.n.01"},{"index":5,"sense_key":"sign%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":6,"sense_key":"sign%1:26:00::","synset_id":"sign.n.06"},{"index":7,"sense_key":"sign%1:24:00::","synset_id":"polarity.n.02"},{"index":8,"sense_key":"sign%1:11:00::","synset_id":"augury.n.01"},{"index":9,"sense_key":"sign%1:10:01::","synset_id":"sign.n.09"},{"index":10,"sense_key":"sign%1:10:06::","synset_id":"sign.n.10"},{"index":11,"sense_key":"sign%1:10:04::","synset_id":"sign.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sign%1:10:03::","call_ids":["senseval2.d000.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sign%1:10:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037844999999999996,"input_uncached_usd":0.00003925,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.504598249972332}
{"item_id":"senseval2.d000.s057.t003","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["senseval2.d000.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035305,"input_uncached_usd":0.00003945,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.391232208989095}
{"item_id":"senseval2.d000.s057.t004","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s057.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046669999999999996,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.660823374986649}
{"item_id":"senseval2.d000.s057.t005","gold_sense_keys":["enter%2:33:00::","enter%2:38:00::"],"candidates":[{"index":1,"sense_key":"enter%2:38:00::","synset_id":"enter.v.01"},{"index":2,"sense_key":"enter%2:33:00::","synset_id":"enter.v.02"},{"index":3,"sense_key":"enter%2:41:06::","synset_id":"enroll.v.01"},{"index":4,"sense_key":"enter%2:42:00::","synset_id":"figure.v.02"},{"index":5,"sense_key":"enter%2:32:04::","synset_id":"record.v.01"},{"index":6,"sense_key":"enter%2:36:00::","synset_id":"enter.v.06"},{"index":7,"sense_key":"enter%2:41:00::","synset_id":"accede.v.02"},{"index":8,"sense_key":"enter%2:35:03::","synset_id":"insert.v.01"},{"index":9,"sense_key":"enter%2:30:10::","synset_id":"embark.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"enter%2:33:00::","call_ids":["senseval2.d000.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"enter%2:33:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021884999999999998,"input_uncached_usd":0.00003325,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.008802208001725}
{"item_id":"senseval2.d000.s057.t006","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d000.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015539999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.124676499981433}
{"item_id":"senseval2.d000.s057.t007","gold_sense_keys":["field%1:09:00::","field%1:26:00::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"field%1:26:00::","call_ids":["senseval2.d000.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"field%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1077,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.00100745,"input_uncached_usd":0.000053849999999999994,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.57085116696544}
{"item_id":"senseval2.d000.s057.t009","gold_sense_keys":["third%1:23:00::"],"candidates":[{"index":1,"sense_key":"third%1:23:00::","synset_id":"one-third.n.01"},{"index":2,"sense_key":"third%1:04:00::","synset_id":"third_base.n.02"},{"index":3,"sense_key":"third%1:24:00::","synset_id":"third.n.03"},{"index":4,"sense_key":"third%1:10:00::","synset_id":"third.n.04"},{"index":5,"sense_key":"third%1:06:00::","synset_id":"third_gear.n.01"},{"index":6,"sense_key":"third%1:06:01::","synset_id":"third_base.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"third%1:23:00::","call_ids":["senseval2.d000.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"third%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024109999999999998,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.298551707994193}
{"item_id":"senseval2.d000.s057.t010","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s057.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020769999999999998,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.957249708997551}
{"item_id":"senseval2.d000.s057.t011","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d000.s057.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005891,"input_uncached_usd":0.0000191,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.56673395796679}
{"item_id":"senseval2.d000.s057.t012","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s057.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005946500000000001,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.77229079202516}
{"item_id":"senseval2.d000.s058.t000","gold_sense_keys":["accept%2:40:03::"],"candidates":[{"index":1,"sense_key":"accept%2:31:00::","synset_id":"accept.v.01"},{"index":2,"sense_key":"accept%2:40:00::","synset_id":"accept.v.02"},{"index":3,"sense_key":"accept%2:32:00::","synset_id":"accept.v.03"},{"index":4,"sense_key":"accept%2:31:01::","synset_id":"accept.v.04"},{"index":5,"sense_key":"accept%2:40:03::","synset_id":"accept.v.05"},{"index":6,"sense_key":"accept%2:40:01::","synset_id":"bear.v.06"},{"index":7,"sense_key":"accept%2:31:03::","synset_id":"accept.v.07"},{"index":8,"sense_key":"accept%2:42:00::","synset_id":"accept.v.08"},{"index":9,"sense_key":"accept%2:40:07::","synset_id":"accept.v.09"},{"index":10,"sense_key":"accept%2:40:12::","synset_id":"take.v.20"},{"index":11,"sense_key":"accept%2:31:08::","synset_id":"accept.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"accept%2:31:03::","call_ids":["senseval2.d000.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"accept%2:31:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":1107,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048449999999999996,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.0004428,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.962864457978867}
{"item_id":"senseval2.d000.s058.t002","gold_sense_keys":["however%4:02:00::"],"candidates":[{"index":1,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":2,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":3,"sense_key":"however%4:02:02::","synset_id":"however.r.03"},{"index":4,"sense_key":"however%4:02:01::","synset_id":"however.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"however%4:02:00::","call_ids":["senseval2.d000.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"however%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031585,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.783575208974071}
{"item_id":"senseval2.d000.s059.t000","gold_sense_keys":["bell_ringing%1:04:00::"],"candidates":[{"index":1,"sense_key":"bell_ringing%1:11:00::","synset_id":"bell_ringing.n.01"},{"index":2,"sense_key":"bell_ringing%1:10:00::","synset_id":"electioneering.n.01"},{"index":3,"sense_key":"bell_ringing%1:04:00::","synset_id":"bell_ringing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bell_ringing%1:04:00::","call_ids":["senseval2.d000.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bell_ringing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015815,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8703447080333717}
{"item_id":"senseval2.d000.s059.t001","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028649999999999997,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.760198042029515}
{"item_id":"senseval2.d000.s059.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["senseval2.d000.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039539999999999996,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.433235000004061}
{"item_id":"senseval2.d000.s059.t003","gold_sense_keys":["found%2:36:00::","found%2:41:00::"],"candidates":[{"index":1,"sense_key":"found%2:41:00::","synset_id":"establish.v.01"},{"index":2,"sense_key":"found%2:36:00::","synset_id":"establish.v.02"},{"index":3,"sense_key":"found%2:31:00::","synset_id":"establish.v.08"},{"index":4,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":5,"sense_key":"detect%2:39:00::","synset_id":"detect.v.01"},{"index":6,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":7,"sense_key":"determine%2:32:00::","synset_id":"determine.v.01"},{"index":8,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":9,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"},{"index":10,"sense_key":"line_up%2:40:00::","synset_id":"line_up.v.02"},{"index":11,"sense_key":"discover%2:36:00::","synset_id":"discover.v.03"},{"index":12,"sense_key":"discover%2:31:00::","synset_id":"discover.v.04"},{"index":13,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":14,"sense_key":"rule%2:32:00::","synset_id":"rule.v.04"},{"index":15,"sense_key":"receive%2:30:01::","synset_id":"receive.v.02"},{"index":16,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":17,"sense_key":"recover%2:40:00::","synset_id":"recover.v.01"},{"index":18,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":19,"sense_key":"find_oneself%2:30:00::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"found%2:41:00::","call_ids":["senseval2.d000.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"found%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1325,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00043105,"input_uncached_usd":0.00006625,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.81345545797376}
{"item_id":"senseval2.d000.s059.t004","gold_sense_keys":["remain%2:30:00::"],"candidates":[{"index":1,"sense_key":"remain%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"remain%2:42:03::","synset_id":"stay.v.04"},{"index":3,"sense_key":"remain%2:42:05::","synset_id":"remain.v.03"},{"index":4,"sense_key":"remain%2:42:01::","synset_id":"persist.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"remain%2:42:03::","call_ids":["senseval2.d000.s059.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"remain%2:42:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013769999999999999,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.654449499968905}
{"item_id":"senseval2.d000.s059.t005","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fact%1:26:01::","call_ids":["senseval2.d000.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fact%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008269,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.401502624968998}
{"item_id":"senseval2.d000.s059.t006","gold_sense_keys":["particularly%4:02:00::"],"candidates":[{"index":1,"sense_key":"particularly%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"particularly%4:02:02::","synset_id":"particularly.r.02"},{"index":3,"sense_key":"particularly%4:02:01::","synset_id":"peculiarly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"particularly%4:02:00::","call_ids":["senseval2.d000.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"particularly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026245,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3446920000133105}
{"item_id":"senseval2.d000.s059.t008","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s059.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":1298,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005472999999999999,"input_uncached_usd":0.0000281,"input_cached_usd":0.0,"output_usd":0.0005191999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.358721541997511}
{"item_id":"senseval2.d000.s059.t009","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["senseval2.d000.s059.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003121,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.790699291974306}
{"item_id":"senseval2.d000.s059.t011","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s059.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021069999999999997,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.032666958053596}
{"item_id":"senseval2.d000.s059.t013","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d000.s059.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001849,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.724753040994983}
{"item_id":"senseval2.d000.s059.t015","gold_sense_keys":["church%1:06:00::"],"candidates":[{"index":1,"sense_key":"church%1:14:00::","synset_id":"church.n.01"},{"index":2,"sense_key":"church%1:06:00::","synset_id":"church.n.02"},{"index":3,"sense_key":"church%1:04:00::","synset_id":"church_service.n.01"},{"index":4,"sense_key":"church%1:14:01::","synset_id":"church.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"church%1:06:00::","call_ids":["senseval2.d000.s059.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"church%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002626,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.187445582996588}
{"item_id":"senseval2.d000.s060.t001","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:14:00::","call_ids":["senseval2.d000.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031725,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.23834712500684}
{"item_id":"senseval2.d000.s060.t002","gold_sense_keys":["file%2:32:02::"],"candidates":[{"index":1,"sense_key":"file%2:32:02::","synset_id":"file.v.01"},{"index":2,"sense_key":"file%2:35:00::","synset_id":"file.v.02"},{"index":3,"sense_key":"file%2:38:00::","synset_id":"file.v.03"},{"index":4,"sense_key":"file%2:32:01::","synset_id":"charge.v.06"},{"index":5,"sense_key":"file%2:32:00::","synset_id":"file.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"file%2:32:02::","call_ids":["senseval2.d000.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"file%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021339999999999998,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.728286541998386}
{"item_id":"senseval2.d000.s060.t004","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":2,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":4,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suit%1:04:00::","call_ids":["senseval2.d000.s060.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014289999999999998,"input_uncached_usd":0.000034099999999999995,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7714807500015013}
{"item_id":"senseval2.d000.s060.t005","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval2.d000.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015785,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.165802416973747}
{"item_id":"senseval2.d000.s060.t006","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["senseval2.d000.s060.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015979999999999998,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.382071874977555}
{"item_id":"senseval2.d000.s060.t007","gold_sense_keys":["surface%2:30:00::"],"candidates":[{"index":1,"sense_key":"surface%2:38:00::","synset_id":"surface.v.01"},{"index":2,"sense_key":"surface%2:35:00::","synset_id":"coat.v.01"},{"index":3,"sense_key":"surface%2:30:00::","synset_id":"come_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"surface%2:30:00::","call_ids":["senseval2.d000.s060.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"surface%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026284999999999997,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.761599750025198}
{"item_id":"senseval2.d000.s060.t008","gold_sense_keys":["summer%1:28:00::"],"candidates":[{"index":1,"sense_key":"summer%1:28:00::","synset_id":"summer.n.01"},{"index":2,"sense_key":"summer%1:28:01::","synset_id":"summer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"summer%1:28:00::","call_ids":["senseval2.d000.s060.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"summer%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046564999999999996,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.381066165980883}
{"item_id":"senseval2.d000.s060.t009","gold_sense_keys":["series%1:14:00::"],"candidates":[{"index":1,"sense_key":"series%1:14:00::","synset_id":"series.n.01"},{"index":2,"sense_key":"series%1:10:01::","synset_id":"serial.n.01"},{"index":3,"sense_key":"series%1:10:00::","synset_id":"series.n.03"},{"index":4,"sense_key":"series%1:11:00::","synset_id":"series.n.04"},{"index":5,"sense_key":"series%1:24:00::","synset_id":"series.n.05"},{"index":6,"sense_key":"series%1:14:01::","synset_id":"series.n.06"},{"index":7,"sense_key":"series%1:09:00::","synset_id":"series.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"series%1:14:00::","call_ids":["senseval2.d000.s060.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"series%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037309999999999996,"input_uncached_usd":0.0000339,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.840909041988198}
{"item_id":"senseval2.d000.s060.t010","gold_sense_keys":["letter%1:10:00::"],"candidates":[{"index":1,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":2,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":3,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":4,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":5,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"letter%1:10:00::","call_ids":["senseval2.d000.s060.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"letter%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003948,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.400630042015109}
{"item_id":"senseval2.d000.s060.t012","gold_sense_keys":["newspaper%1:10:00::"],"candidates":[{"index":1,"sense_key":"newspaper%1:10:00::","synset_id":"newspaper.n.01"},{"index":2,"sense_key":"newspaper%1:14:00::","synset_id":"newspaper.n.02"},{"index":3,"sense_key":"newspaper%1:06:00::","synset_id":"newspaper.n.03"},{"index":4,"sense_key":"newspaper%1:27:00::","synset_id":"newspaper.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"newspaper%1:10:00::","call_ids":["senseval2.d000.s060.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"newspaper%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001368,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.630005125014577}
{"item_id":"senseval2.d000.s060.t013","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s060.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004938,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.812051291984972}
{"item_id":"senseval2.d000.s061.t000","gold_sense_keys":["writer%1:18:01::"],"candidates":[{"index":1,"sense_key":"writer%1:18:00::","synset_id":"writer.n.01"},{"index":2,"sense_key":"writer%1:18:01::","synset_id":"writer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"writer%1:18:01::","call_ids":["senseval2.d000.s061.t000__v1__a1","senseval2.d000.s061.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"writer%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00070385,"input_uncached_usd":0.000051049999999999994,"input_cached_usd":0.0,"output_usd":0.0006527999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.321646958007477}
{"item_id":"senseval2.d000.s061.t001","gold_sense_keys":["sign%2:32:02::"],"candidates":[{"index":1,"sense_key":"sign%2:32:02::","synset_id":"sign.v.01"},{"index":2,"sense_key":"sign%2:41:00::","synset_id":"sign.v.02"},{"index":3,"sense_key":"sign%2:32:00::","synset_id":"sign.v.03"},{"index":4,"sense_key":"sign%2:41:01::","synset_id":"sign.v.04"},{"index":5,"sense_key":"sign%2:32:01::","synset_id":"sign.v.05"},{"index":6,"sense_key":"sign%2:35:02::","synset_id":"sign.v.06"},{"index":7,"sense_key":"sign%2:32:04::","synset_id":"sign.v.07"},{"index":8,"sense_key":"sign%2:32:03::","synset_id":"bless.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sign%2:32:02::","call_ids":["senseval2.d000.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sign%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037889999999999994,"input_uncached_usd":0.000039699999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.297054416965693}
{"item_id":"senseval2.d000.s061.t002","gold_sense_keys":["letter%1:10:00::"],"candidates":[{"index":1,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":2,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":3,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":4,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":5,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"letter%1:10:00::","call_ids":["senseval2.d000.s061.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"letter%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014025,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7676467500277795}
{"item_id":"senseval2.d000.s061.t005","gold_sense_keys":["male%1:18:00::"],"candidates":[{"index":1,"sense_key":"male%1:05:00::","synset_id":"male.n.01"},{"index":2,"sense_key":"male%1:18:00::","synset_id":"male.n.02"},{"index":3,"sense_key":"male%1:15:00::","synset_id":"male.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"male%1:18:00::","call_ids":["senseval2.d000.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"male%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041685,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.490466624964029}
{"item_id":"senseval2.d000.s061.t006","gold_sense_keys":["remark%2:32:01::"],"candidates":[{"index":1,"sense_key":"remark%2:32:00::","synset_id":"note.v.01"},{"index":2,"sense_key":"remark%2:32:01::","synset_id":"comment.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"remark%2:32:01::","call_ids":["senseval2.d000.s061.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"remark%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002371,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.145640125032514}
{"item_id":"senseval2.d000.s061.t007","gold_sense_keys":["frequency%1:28:00::"],"candidates":[{"index":1,"sense_key":"frequency%1:28:00::","synset_id":"frequency.n.01"},{"index":2,"sense_key":"frequency%1:24:00::","synset_id":"frequency.n.02"},{"index":3,"sense_key":"frequency%1:23:00::","synset_id":"frequency.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"frequency%1:28:00::","call_ids":["senseval2.d000.s061.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"frequency%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00075075,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.453748958010692}
{"item_id":"senseval2.d000.s061.t008","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:14:00::","call_ids":["senseval2.d000.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007799,"input_uncached_usd":0.0000311,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.204914583999198}
{"item_id":"senseval2.d000.s061.t010","gold_sense_keys":["suggest%2:32:00::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suggest%2:32:00::","call_ids":["senseval2.d000.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suggest%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002163,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.583855458011385}
{"item_id":"senseval2.d000.s061.t012","gold_sense_keys":["back%4:02:05::"],"candidates":[{"index":1,"sense_key":"back%4:02:04::","synset_id":"back.r.01"},{"index":2,"sense_key":"back%4:02:00::","synset_id":"back.r.02"},{"index":3,"sense_key":"back%4:02:05::","synset_id":"back.r.03"},{"index":4,"sense_key":"back%4:02:01::","synset_id":"back.r.04"},{"index":5,"sense_key":"back%4:02:06::","synset_id":"back.r.05"},{"index":6,"sense_key":"back%4:02:03::","synset_id":"back.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"back%4:02:05::","call_ids":["senseval2.d000.s061.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"back%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001925,"input_uncached_usd":0.0000325,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.572013667027932}
{"item_id":"senseval2.d000.s061.t013","gold_sense_keys":["traditional%3:00:00::"],"candidates":[{"index":1,"sense_key":"traditional%3:00:00::","synset_id":"traditional.a.01"},{"index":2,"sense_key":"traditional%5:00:00:orthodox:00","synset_id":"traditional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"traditional%3:00:00::","call_ids":["senseval2.d000.s061.t013__v1__a1","senseval2.d000.s061.t013__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"traditional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00062805,"input_uncached_usd":0.00005204999999999999,"input_cached_usd":0.0,"output_usd":0.0005759999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.140190917009022}
{"item_id":"senseval2.d000.s061.t014","gold_sense_keys":["role%1:04:00::","role%1:04:01::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"role%1:07:00::","call_ids":["senseval2.d000.s061.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"role%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006772,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.729850417003036}
{"item_id":"senseval2.d000.s061.t015","gold_sense_keys":["make%2:36:07::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":39,"chosen_sense_key":"make%2:36:07::","call_ids":["senseval2.d000.s061.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":39,"predicted_sense_key":"make%2:36:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2381,"cached_input_tokens":0,"output_tokens":1103,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00056025,"input_uncached_usd":0.00011905,"input_cached_usd":0.0,"output_usd":0.0004412,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.270435958984308}
{"item_id":"senseval2.d000.s061.t016","gold_sense_keys":["tea%1:13:00::"],"candidates":[{"index":1,"sense_key":"tea%1:13:00::","synset_id":"tea.n.01"},{"index":2,"sense_key":"tea%1:13:01::","synset_id":"tea.n.02"},{"index":3,"sense_key":"tea%1:20:00::","synset_id":"tea.n.03"},{"index":4,"sense_key":"tea%1:14:00::","synset_id":"tea.n.04"},{"index":5,"sense_key":"tea%1:13:02::","synset_id":"tea.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tea%1:13:00::","call_ids":["senseval2.d000.s061.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tea%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047439999999999993,"input_uncached_usd":0.0000328,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.204227707989048}
{"item_id":"senseval2.d000.s061.t017","gold_sense_keys":["meeting%1:14:00::","meeting%1:14:01::"],"candidates":[{"index":1,"sense_key":"meeting%1:14:00::","synset_id":"meeting.n.01"},{"index":2,"sense_key":"meeting%1:14:01::","synset_id":"meeting.n.02"},{"index":3,"sense_key":"meeting%1:11:00::","synset_id":"meeting.n.03"},{"index":4,"sense_key":"meeting%1:04:00::","synset_id":"meeting.n.04"},{"index":5,"sense_key":"meeting%1:04:02::","synset_id":"merging.n.01"},{"index":6,"sense_key":"meeting%1:15:00::","synset_id":"confluence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"meeting%1:04:00::","call_ids":["senseval2.d000.s061.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"meeting%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045139999999999997,"input_uncached_usd":0.000035,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.421470207977109}
{"item_id":"senseval2.d000.s062.t000","gold_sense_keys":["torrent%1:23:00::"],"candidates":[{"index":1,"sense_key":"torrent%1:19:01::","synset_id":"downpour.n.01"},{"index":2,"sense_key":"torrent%1:11:00::","synset_id":"torrent.n.02"},{"index":3,"sense_key":"torrent%1:23:00::","synset_id":"flood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"torrent%1:23:00::","call_ids":["senseval2.d000.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"torrent%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011099999999999999,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5581024999846704}
{"item_id":"senseval2.d000.s062.t001","gold_sense_keys":["reply%1:10:01::"],"candidates":[{"index":1,"sense_key":"reply%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"reply%1:10:00::","synset_id":"reply.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reply%1:10:01::","call_ids":["senseval2.d000.s062.t001__v1__a1","senseval2.d000.s062.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reply%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1107,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006569499999999999,"input_uncached_usd":0.000055350000000000004,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.793627125036437}
{"item_id":"senseval2.d000.s062.t002","gold_sense_keys":["follow%2:42:00::","follow%2:42:01::","follow%2:42:05::"],"candidates":[{"index":1,"sense_key":"follow%2:38:00::","synset_id":"follow.v.01"},{"index":2,"sense_key":"follow%2:42:00::","synset_id":"postdate.v.01"},{"index":3,"sense_key":"follow%2:42:05::","synset_id":"follow.v.03"},{"index":4,"sense_key":"follow%2:38:01::","synset_id":"follow.v.04"},{"index":5,"sense_key":"follow%2:41:00::","synset_id":"comply.v.01"},{"index":6,"sense_key":"follow%2:42:01::","synset_id":"follow.v.06"},{"index":7,"sense_key":"follow%2:30:00::","synset_id":"follow.v.07"},{"index":8,"sense_key":"follow%2:42:02::","synset_id":"follow.v.08"},{"index":9,"sense_key":"follow%2:40:00::","synset_id":"adopt.v.01"},{"index":10,"sense_key":"follow%2:42:03::","synset_id":"follow.v.10"},{"index":11,"sense_key":"follow%2:36:00::","synset_id":"take_after.v.02"},{"index":12,"sense_key":"follow%2:31:00::","synset_id":"trace.v.01"},{"index":13,"sense_key":"follow%2:41:02::","synset_id":"watch.v.02"},{"index":14,"sense_key":"follow%2:41:01::","synset_id":"succeed.v.02"},{"index":15,"sense_key":"follow%2:36:01::","synset_id":"play_along.v.02"},{"index":16,"sense_key":"follow%2:30:01::","synset_id":"keep_up.v.04"},{"index":17,"sense_key":"follow%2:42:04::","synset_id":"come.v.05"},{"index":18,"sense_key":"follow%2:41:10::","synset_id":"follow.v.18"},{"index":19,"sense_key":"follow%2:41:11::","synset_id":"follow.v.19"},{"index":20,"sense_key":"follow%2:41:09::","synset_id":"be.v.08"},{"index":21,"sense_key":"follow%2:39:13::","synset_id":"surveil.v.01"},{"index":22,"sense_key":"follow%2:38:13::","synset_id":"pursue.v.02"},{"index":23,"sense_key":"follow%2:31:14::","synset_id":"follow.v.23"},{"index":24,"sense_key":"follow%2:30:12::","synset_id":"stick_to.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"follow%2:42:01::","call_ids":["senseval2.d000.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"follow%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1477,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00061785,"input_uncached_usd":0.00007384999999999999,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.679837416973896}
{"item_id":"senseval2.d000.s062.t003","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000217,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.486296082963236}
{"item_id":"senseval2.d000.s062.t004","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ringer%1:18:00::","call_ids":["senseval2.d000.s062.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ringer%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039315,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.685345125035383}
{"item_id":"senseval2.d000.s062.t005","gold_sense_keys":["observe%2:32:00::"],"candidates":[{"index":1,"sense_key":"observe%2:39:02::","synset_id":"detect.v.01"},{"index":2,"sense_key":"observe%2:32:00::","synset_id":"note.v.01"},{"index":3,"sense_key":"observe%2:39:01::","synset_id":"note.v.03"},{"index":4,"sense_key":"observe%2:39:00::","synset_id":"observe.v.04"},{"index":5,"sense_key":"observe%2:41:01::","synset_id":"respect.v.02"},{"index":6,"sense_key":"observe%2:41:02::","synset_id":"observe.v.06"},{"index":7,"sense_key":"observe%2:41:00::","synset_id":"watch.v.02"},{"index":8,"sense_key":"observe%2:31:00::","synset_id":"observe.v.08"},{"index":9,"sense_key":"observe%2:41:04::","synset_id":"observe.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"observe%2:32:00::","call_ids":["senseval2.d000.s062.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"observe%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005089,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.90200958296191}
{"item_id":"senseval2.d000.s062.t006","gold_sense_keys":["average%5:00:00:common:01"],"candidates":[{"index":1,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":2,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":3,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":6,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"average%5:00:00:ordinary:00","call_ids":["senseval2.d000.s062.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"average%5:00:00:ordinary:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022444999999999998,"input_uncached_usd":0.000038849999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.244520250009373}
{"item_id":"senseval2.d000.s062.t007","gold_sense_keys":["male%3:00:00::"],"candidates":[{"index":1,"sense_key":"male%3:00:00::","synset_id":"male.a.01"},{"index":2,"sense_key":"male%5:00:02:masculine:01","synset_id":"male.s.01"},{"index":3,"sense_key":"male%5:00:01:male:00","synset_id":"male.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"male%5:00:01:male:00","call_ids":["senseval2.d000.s062.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"male%5:00:01:male:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011204999999999999,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.526832709030714}
{"item_id":"senseval2.d000.s062.t008","gold_sense_keys":["ringer%1:18:02::"],"candidates":[{"index":1,"sense_key":"ringer%1:18:02::","synset_id":"toller.n.01"},{"index":2,"sense_key":"ringer%1:18:01::","synset_id":"ringer.n.02"},{"index":3,"sense_key":"ringer%1:18:00::","synset_id":"ringer.n.03"},{"index":4,"sense_key":"ringer%1:04:00::","synset_id":"ringer.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ringer%1:18:02::","call_ids":["senseval2.d000.s062.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ringer%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016274999999999997,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8864617079962045}
{"item_id":"senseval2.d000.s062.t013","gold_sense_keys":["badly%4:02:00::","badly%4:02:12::"],"candidates":[{"index":1,"sense_key":"badly%4:02:02::","synset_id":"badly.r.01"},{"index":2,"sense_key":"badly%4:02:00::","synset_id":"ill.r.01"},{"index":3,"sense_key":"badly%4:02:06::","synset_id":"badly.r.03"},{"index":4,"sense_key":"badly%4:02:05::","synset_id":"badly.r.04"},{"index":5,"sense_key":"badly%4:02:04::","synset_id":"badly.r.05"},{"index":6,"sense_key":"badly%4:02:03::","synset_id":"badly.r.06"},{"index":7,"sense_key":"badly%4:02:12::","synset_id":"badly.r.07"},{"index":8,"sense_key":"badly%4:02:10::","synset_id":"badly.r.08"},{"index":9,"sense_key":"badly%4:02:07::","synset_id":"ill.r.02"},{"index":10,"sense_key":"badly%4:02:08::","synset_id":"badly.r.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"badly%4:02:00::","call_ids":["senseval2.d000.s062.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"badly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":925,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038585,"input_uncached_usd":0.00004625,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.564318165997975}
{"item_id":"senseval2.d000.s062.t014","gold_sense_keys":["dressed%5:00:00:clothed:00","dressed%5:00:02:clothed:00"],"candidates":[{"index":1,"sense_key":"dressed%5:00:00:clothed:00","synset_id":"appareled.s.01"},{"index":2,"sense_key":"dressed%5:00:00:treated:02","synset_id":"dressed.s.02"},{"index":3,"sense_key":"dressed%5:00:00:finished:02","synset_id":"dressed.s.03"},{"index":4,"sense_key":"dressed%5:00:02:clothed:00","synset_id":"dressed.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dressed%5:00:00:clothed:00","call_ids":["senseval2.d000.s062.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dressed%5:00:00:clothed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005747,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.773833499988541}
{"item_id":"senseval2.d000.s062.t017","gold_sense_keys":["large%3:00:00::"],"candidates":[{"index":1,"sense_key":"large%3:00:00::","synset_id":"large.a.01"},{"index":2,"sense_key":"large%5:00:00:significant:00","synset_id":"large.s.01"},{"index":3,"sense_key":"large%5:00:00:rhetorical:00","synset_id":"bombastic.s.01"},{"index":4,"sense_key":"large%5:00:00:generous:02","synset_id":"big.s.10"},{"index":5,"sense_key":"large%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"large%5:00:00:comprehensive:00","synset_id":"large.s.05"},{"index":7,"sense_key":"large%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"large%3:00:00::","call_ids":["senseval2.d000.s062.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"large%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017294999999999998,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.60671383398585}
{"item_id":"senseval2.d000.s062.t019","gold_sense_keys":["unwashed%5:00:00:dirty:01"],"candidates":[{"index":1,"sense_key":"unwashed%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":2,"sense_key":"unwashed%5:00:00:dirty:01","synset_id":"unwashed.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unwashed%5:00:00:dirty:01","call_ids":["senseval2.d000.s062.t019__v1__a1","senseval2.d000.s062.t019__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unwashed%5:00:00:dirty:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1109,"cached_input_tokens":0,"output_tokens":1888,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00081065,"input_uncached_usd":0.00005544999999999999,"input_cached_usd":0.0,"output_usd":0.0007551999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.716935292002745}
{"item_id":"senseval2.d000.s063.t000","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval2.d000.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019045,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.451374999945983}
{"item_id":"senseval2.d000.s063.t001","gold_sense_keys":["write%2:32:01::"],"candidates":[{"index":1,"sense_key":"write%2:36:00::","synset_id":"write.v.01"},{"index":2,"sense_key":"write%2:32:00::","synset_id":"write.v.02"},{"index":3,"sense_key":"write%2:36:02::","synset_id":"publish.v.03"},{"index":4,"sense_key":"write%2:32:08::","synset_id":"write.v.04"},{"index":5,"sense_key":"write%2:32:01::","synset_id":"write.v.05"},{"index":6,"sense_key":"write%2:36:04::","synset_id":"compose.v.02"},{"index":7,"sense_key":"write%2:36:01::","synset_id":"write.v.07"},{"index":8,"sense_key":"write%2:32:03::","synset_id":"write.v.08"},{"index":9,"sense_key":"write%2:36:03::","synset_id":"spell.v.03"},{"index":10,"sense_key":"write%2:36:10::","synset_id":"write.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"write%2:32:01::","call_ids":["senseval2.d000.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"write%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002779,"input_uncached_usd":0.0000407,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.557255916995928}
{"item_id":"senseval2.d000.s063.t002","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d000.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004095,"input_uncached_usd":0.000044699999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.024815500015393}
{"item_id":"senseval2.d000.s063.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d000.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029275,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.932161708013155}
{"item_id":"senseval2.d000.s063.t004","gold_sense_keys":["never%4:02:00::"],"candidates":[{"index":1,"sense_key":"never%4:02:00::","synset_id":"never.r.01"},{"index":2,"sense_key":"never%4:02:01::","synset_id":"never.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"never%4:02:00::","call_ids":["senseval2.d000.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"never%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.000569,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.553969624976162}
{"item_id":"senseval2.d000.s063.t005","gold_sense_keys":["know%2:31:04::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"know%2:31:04::","call_ids":["senseval2.d000.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"know%2:31:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038549999999999994,"input_uncached_usd":0.0000463,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.269986875005998}
{"item_id":"senseval2.d000.s063.t006","gold_sense_keys":["lady%1:18:02::"],"candidates":[{"index":1,"sense_key":"lady%1:18:02::","synset_id":"lady.n.01"},{"index":2,"sense_key":"lady%1:18:01::","synset_id":"dame.n.02"},{"index":3,"sense_key":"lady%1:18:00::","synset_id":"lady.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lady%1:18:02::","call_ids":["senseval2.d000.s063.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lady%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006469000000000001,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.257646375044715}
{"item_id":"senseval2.d000.s063.t008","gold_sense_keys":["belfry%1:06:00::"],"candidates":[{"index":1,"sense_key":"belfry%1:06:01::","synset_id":"campanile.n.01"},{"index":2,"sense_key":"belfry%1:06:00::","synset_id":"belfry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"belfry%1:06:00::","call_ids":["senseval2.d000.s063.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"belfry%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":2128,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.0008752,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.0008512,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.725688833976164}
{"item_id":"senseval2.d000.s064.t000","gold_sense_keys":["see%2:39:00::","see%2:39:02::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"see%2:39:02::","call_ids":["senseval2.d000.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"see%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1547,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00039095,"input_uncached_usd":0.00007735,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.4686535830260254}
{"item_id":"senseval2.d000.s064.t001","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval2.d000.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":940,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004374,"input_uncached_usd":0.000047,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.072810833982658}
{"item_id":"senseval2.d000.s064.t002","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":2,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":3,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":4,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":5,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":6,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":7,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":8,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":9,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":10,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":11,"sense_key":"die%2:30:02::","synset_id":"die.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval2.d000.s064.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025475,"input_uncached_usd":0.000043549999999999995,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.742269292008132}
{"item_id":"senseval2.d001.s000.t000","gold_sense_keys":["medical%3:01:00::"],"candidates":[{"index":1,"sense_key":"medical%3:01:00::","synset_id":"medical.a.01"},{"index":2,"sense_key":"medical%3:00:00::","synset_id":"medical.a.02"},{"index":3,"sense_key":"medical%3:01:02::","synset_id":"aesculapian.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medical%3:01:00::","call_ids":["senseval2.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":310,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002523,"input_uncached_usd":0.0000155,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.573252500034869}
{"item_id":"senseval2.d001.s000.t002","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval2.d001.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00063635,"input_uncached_usd":0.00004115,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.010029166005552}
{"item_id":"senseval2.d001.s000.t003","gold_sense_keys":["handful%1:23:01::"],"candidates":[{"index":1,"sense_key":"handful%1:23:01::","synset_id":"handful.n.01"},{"index":2,"sense_key":"handful%1:23:00::","synset_id":"handful.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"handful%1:23:01::","call_ids":["senseval2.d001.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"handful%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":266,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003013,"input_uncached_usd":0.0000133,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.016492458991706}
{"item_id":"senseval2.d001.s000.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":283,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00068615,"input_uncached_usd":0.00001415,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.833337374962866}
{"item_id":"senseval2.d001.s000.t006","gold_sense_keys":["unleash%2:35:02::"],"candidates":[{"index":1,"sense_key":"unleash%2:35:00::","synset_id":"unleash.v.01"},{"index":2,"sense_key":"unleash%2:35:01::","synset_id":"unleash.v.02"},{"index":3,"sense_key":"unleash%2:35:02::","synset_id":"unleash.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"unleash%2:35:02::","call_ids":["senseval2.d001.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"unleash%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":307,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020094999999999998,"input_uncached_usd":0.00001535,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.015090083004907}
{"item_id":"senseval2.d001.s000.t007","gold_sense_keys":["chaotic%5:00:00:wild:02"],"candidates":[{"index":1,"sense_key":"chaotic%5:00:00:disorganized:00","synset_id":"chaotic.s.01"},{"index":2,"sense_key":"chaotic%5:00:00:wild:02","synset_id":"chaotic.s.02"},{"index":3,"sense_key":"chaotic%3:01:00::","synset_id":"chaotic.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"chaotic%5:00:00:wild:02","call_ids":["senseval2.d001.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"chaotic%5:00:00:wild:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":294,"cached_input_tokens":0,"output_tokens":2512,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.0010195,"input_uncached_usd":0.0000147,"input_cached_usd":0.0,"output_usd":0.0010048,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.617650250031147}
{"item_id":"senseval2.d001.s000.t008","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"growth%1:26:00::","call_ids":["senseval2.d001.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"growth%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026125,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.353466875036247}
{"item_id":"senseval2.d001.s000.t009","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028585,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.4233731660060585}
{"item_id":"senseval2.d001.s000.t010","gold_sense_keys":["characterize%2:42:01::"],"candidates":[{"index":1,"sense_key":"characterize%2:32:00::","synset_id":"qualify.v.06"},{"index":2,"sense_key":"characterize%2:42:01::","synset_id":"characterize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"characterize%2:42:01::","call_ids":["senseval2.d001.s000.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"characterize%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":298,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008405,"input_uncached_usd":0.0000149,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.50183962500887}
{"item_id":"senseval2.d001.s000.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s000.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020539999999999998,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.222290958976373}
{"item_id":"senseval2.d001.s001.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"say%2:32:13::","call_ids":["senseval2.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"say%2:32:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006818499999999999,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.981603041989729}
{"item_id":"senseval2.d001.s001.t002","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":348,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048459999999999996,"input_uncached_usd":0.0000174,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.62075191596523}
{"item_id":"senseval2.d001.s001.t004","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recent%5:00:00:new:00","call_ids":["senseval2.d001.s001.t004__v1__a1","senseval2.d001.s001.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recent%5:00:00:new:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":3232,"reasoning_output_tokens":3200},"cost":{"currency":"USD","total_usd":0.00132455,"input_uncached_usd":0.00003175,"input_cached_usd":0.0,"output_usd":0.0012927999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":24.808789542061277}
{"item_id":"senseval2.d001.s001.t005","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":294,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004051,"input_uncached_usd":0.0000147,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.168793499993626}
{"item_id":"senseval2.d001.s001.t007","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037429999999999994,"input_uncached_usd":0.0000351,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.139390667027328}
{"item_id":"senseval2.d001.s001.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018094999999999998,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.804424749978352}
{"item_id":"senseval2.d001.s001.t010","gold_sense_keys":["develop%2:30:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"develop%2:30:00::","call_ids":["senseval2.d001.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"develop%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1159,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00032034999999999995,"input_uncached_usd":0.00005795,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.24331766704563}
{"item_id":"senseval2.d001.s002.t001","gold_sense_keys":["understanding%1:09:01::"],"candidates":[{"index":1,"sense_key":"understanding%1:09:01::","synset_id":"understanding.n.01"},{"index":2,"sense_key":"understanding%1:10:00::","synset_id":"agreement.n.01"},{"index":3,"sense_key":"understanding%1:09:03::","synset_id":"sympathy.n.01"},{"index":4,"sense_key":"understanding%1:09:00::","synset_id":"reason.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"understanding%1:09:01::","call_ids":["senseval2.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"understanding%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000259,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.215415208018385}
{"item_id":"senseval2.d001.s002.t003","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval2.d001.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005942499999999999,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.180269500007853}
{"item_id":"senseval2.d001.s002.t004","gold_sense_keys":["produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"produce%2:36:03::","call_ids":["senseval2.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"produce%2:36:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001626,"input_uncached_usd":0.000028199999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.820009791990742}
{"item_id":"senseval2.d001.s002.t005","gold_sense_keys":["array%1:10:00::"],"candidates":[{"index":1,"sense_key":"array%1:14:00::","synset_id":"array.n.01"},{"index":2,"sense_key":"array%1:10:00::","synset_id":"array.n.02"},{"index":3,"sense_key":"array%1:06:01::","synset_id":"array.n.03"},{"index":4,"sense_key":"array%1:06:00::","synset_id":"array.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"array%1:10:00::","call_ids":["senseval2.d001.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"array%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045914999999999997,"input_uncached_usd":0.00001755,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.87474908301374}
{"item_id":"senseval2.d001.s002.t006","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037459999999999994,"input_uncached_usd":0.0000354,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9438482919940725}
{"item_id":"senseval2.d001.s002.t007","gold_sense_keys":["strategy%1:09:00::"],"candidates":[{"index":1,"sense_key":"strategy%1:09:00::","synset_id":"scheme.n.01"},{"index":2,"sense_key":"strategy%1:09:01::","synset_id":"strategy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strategy%1:09:00::","call_ids":["senseval2.d001.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strategy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":287,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012314999999999998,"input_uncached_usd":0.00001435,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4205265000346117}
{"item_id":"senseval2.d001.s002.t008","gold_sense_keys":["future%3:00:00::","future%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"future%3:00:00::","synset_id":"future.a.01"},{"index":2,"sense_key":"future%5:00:00:prospective:00","synset_id":"future.s.01"},{"index":3,"sense_key":"future%5:00:00:incoming:00","synset_id":"future.s.02"},{"index":4,"sense_key":"future%3:01:00::","synset_id":"future.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%3:00:00::","call_ids":["senseval2.d001.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00066495,"input_uncached_usd":0.00001855,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.775262416980695}
{"item_id":"senseval2.d001.s002.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041165,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.3770692920079455}
{"item_id":"senseval2.d001.s002.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["senseval2.d001.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018004999999999999,"input_uncached_usd":0.00002005,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.928904041997157}
{"item_id":"senseval2.d001.s003.t000","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":2,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%1:28:00::","call_ids":["senseval2.d001.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001781,"input_uncached_usd":0.0000177,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.522581417055335}
{"item_id":"senseval2.d001.s004.t002","gold_sense_keys":["develop%2:36:01::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"develop%2:36:01::","call_ids":["senseval2.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"develop%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1235,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00042655,"input_uncached_usd":0.00006175,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.107260082964785}
{"item_id":"senseval2.d001.s004.t004","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["senseval2.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001548,"input_uncached_usd":0.000020399999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.666118917055428}
{"item_id":"senseval2.d001.s004.t006","gold_sense_keys":["identify%2:31:01::"],"candidates":[{"index":1,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":2,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":3,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":4,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"},{"index":5,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":6,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"identify%2:31:01::","call_ids":["senseval2.d001.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"identify%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005715,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.13846604101127}
{"item_id":"senseval2.d001.s004.t008","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d001.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.000418,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.3640338749974035}
{"item_id":"senseval2.d001.s004.t009","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval2.d001.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016925,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7742622090154327}
{"item_id":"senseval2.d001.s004.t011","gold_sense_keys":["otherwise%4:02:01::"],"candidates":[{"index":1,"sense_key":"otherwise%4:02:01::","synset_id":"otherwise.r.01"},{"index":2,"sense_key":"otherwise%4:02:03::","synset_id":"differently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"otherwise%4:02:01::","call_ids":["senseval2.d001.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"otherwise%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046009999999999996,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.888125500001479}
{"item_id":"senseval2.d001.s004.t012","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003915,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.584187207976356}
{"item_id":"senseval2.d001.s004.t013","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s004.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022894999999999998,"input_uncached_usd":0.000017749999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9782607079832815}
{"item_id":"senseval2.d001.s004.t014","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"likely%3:00:04::","call_ids":["senseval2.d001.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"likely%3:00:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020854999999999997,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9159323750063777}
{"item_id":"senseval2.d001.s004.t015","gold_sense_keys":["get%2:29:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"get%2:29:00::","call_ids":["senseval2.d001.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"get%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1926,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004867,"input_uncached_usd":0.0000963,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.073242250015028}
{"item_id":"senseval2.d001.s004.t016","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018465,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.003317875030916}
{"item_id":"senseval2.d001.s005.t000","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":2,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":3,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":4,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":5,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":6,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":7,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":8,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":9,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":10,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":11,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":12,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":13,"sense_key":"set%1:06:01::","synset_id":"set.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%1:14:00::","call_ids":["senseval2.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020395,"input_uncached_usd":0.000043549999999999995,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.292154499969911}
{"item_id":"senseval2.d001.s005.t001","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004893,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.127976791001856}
{"item_id":"senseval2.d001.s005.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval2.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006349000000000001,"input_uncached_usd":0.000039699999999999996,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.186046749993693}
{"item_id":"senseval2.d001.s005.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00054805,"input_uncached_usd":0.000055249999999999994,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.227358750009444}
{"item_id":"senseval2.d001.s006.t000","gold_sense_keys":["only%4:02:05::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"only%4:02:05::","call_ids":["senseval2.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"only%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024094999999999997,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8774428329779766}
{"item_id":"senseval2.d001.s006.t001","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval2.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038465,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.142345042026136}
{"item_id":"senseval2.d001.s006.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013465,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7944407500326633}
{"item_id":"senseval2.d001.s006.t004","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval2.d001.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030399999999999996,"input_uncached_usd":0.000041599999999999995,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.144585625035688}
{"item_id":"senseval2.d001.s006.t006","gold_sense_keys":["molecular%3:01:00::"],"candidates":[{"index":1,"sense_key":"molecular%3:01:00::","synset_id":"molecular.a.01"},{"index":2,"sense_key":"molecular%3:00:00::","synset_id":"molecular.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"molecular%3:01:00::","call_ids":["senseval2.d001.s006.t006__v1__a1","senseval2.d001.s006.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"molecular%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":1185,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00051625,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.000474,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.340488042042125}
{"item_id":"senseval2.d001.s006.t007","gold_sense_keys":["level%1:09:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"level%1:26:00::","call_ids":["senseval2.d001.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"level%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039864999999999996,"input_uncached_usd":0.000033849999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.095344792003743}
{"item_id":"senseval2.d001.s007.t000","gold_sense_keys":["today%1:28:00::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["senseval2.d001.s007.t000__v1__a1","senseval2.d001.s007.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":1313,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056655,"input_uncached_usd":0.00004135,"input_cached_usd":0.0,"output_usd":0.0005252,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.890615958080161}
{"item_id":"senseval2.d001.s007.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval2.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007902,"input_uncached_usd":0.0000414,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.386714208056219}
{"item_id":"senseval2.d001.s007.t002","gold_sense_keys":["accumulation%1:22:00::"],"candidates":[{"index":1,"sense_key":"accumulation%1:22:00::","synset_id":"accretion.n.01"},{"index":2,"sense_key":"accumulation%1:14:00::","synset_id":"collection.n.01"},{"index":3,"sense_key":"accumulation%1:04:00::","synset_id":"accumulation.n.03"},{"index":4,"sense_key":"accumulation%1:21:00::","synset_id":"accumulation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"accumulation%1:04:00::","call_ids":["senseval2.d001.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"accumulation%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025955,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9730795829673298}
{"item_id":"senseval2.d001.s007.t003","gold_sense_keys":["altered%3:00:00::"],"candidates":[{"index":1,"sense_key":"altered%3:00:00::","synset_id":"altered.a.01"},{"index":2,"sense_key":"altered%5:00:00:castrated:00","synset_id":"altered.s.01"},{"index":3,"sense_key":"altered%5:00:00:modified:00","synset_id":"adapted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"altered%3:00:00::","call_ids":["senseval2.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"altered%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033664999999999997,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.010626750008669}
{"item_id":"senseval2.d001.s007.t005","gold_sense_keys":["initiate%2:36:01::"],"candidates":[{"index":1,"sense_key":"initiate%2:36:01::","synset_id":"originate.v.02"},{"index":2,"sense_key":"initiate%2:36:00::","synset_id":"initiate.v.02"},{"index":3,"sense_key":"initiate%2:41:01::","synset_id":"initiate.v.03"},{"index":4,"sense_key":"initiate%2:32:00::","synset_id":"broach.v.01"},{"index":5,"sense_key":"initiate%2:30:00::","synset_id":"lead_up.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"initiate%2:36:01::","call_ids":["senseval2.d001.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"initiate%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028859999999999997,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7764660000102594}
{"item_id":"senseval2.d001.s007.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001857,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9630857500014827}
{"item_id":"senseval2.d001.s007.t007","gold_sense_keys":["then%4:02:00::","then%4:02:02::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033565,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8807674999698065}
{"item_id":"senseval2.d001.s007.t008","gold_sense_keys":["propel%2:35:00::"],"candidates":[{"index":1,"sense_key":"propel%2:35:00::","synset_id":"propel.v.01"},{"index":2,"sense_key":"propel%2:36:00::","synset_id":"motivate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"propel%2:35:00::","call_ids":["senseval2.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"propel%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048715,"input_uncached_usd":0.00001995,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.496329916990362}
{"item_id":"senseval2.d001.s007.t009","gold_sense_keys":["deadly%5:00:01:fatal:00"],"candidates":[{"index":1,"sense_key":"deadly%5:00:01:fatal:00","synset_id":"deadly.s.01"},{"index":2,"sense_key":"deadly%5:00:02:fatal:00","synset_id":"deadly.s.02"},{"index":3,"sense_key":"deadly%5:00:00:toxic:00","synset_id":"deadly.s.03"},{"index":4,"sense_key":"deadly%5:00:00:unpardonable:00","synset_id":"deadly.s.04"},{"index":5,"sense_key":"deadly%5:00:00:noxious:00","synset_id":"baneful.s.01"},{"index":6,"sense_key":"deadly%5:00:00:virulent:00","synset_id":"deadly.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"deadly%5:00:01:fatal:00","call_ids":["senseval2.d001.s007.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"deadly%5:00:01:fatal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002651,"input_uncached_usd":0.0000283,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.098805624991655}
{"item_id":"senseval2.d001.s007.t010","gold_sense_keys":["state%1:03:00::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"state%1:03:00::","call_ids":["senseval2.d001.s007.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"state%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002208,"input_uncached_usd":0.0000352,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1850767500000075}
{"item_id":"senseval2.d001.s008.t001","gold_sense_keys":["call%2:32:00::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"call%2:32:00::","call_ids":["senseval2.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"call%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1485,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00038784999999999997,"input_uncached_usd":0.00007425,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.113978000008501}
{"item_id":"senseval2.d001.s008.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027634999999999997,"input_uncached_usd":0.00003955,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.64138341700891}
{"item_id":"senseval2.d001.s008.t003","gold_sense_keys":["class%1:14:00::"],"candidates":[{"index":1,"sense_key":"class%1:14:00::","synset_id":"class.n.01"},{"index":2,"sense_key":"class%1:14:03::","synset_id":"class.n.02"},{"index":3,"sense_key":"class%1:14:01::","synset_id":"class.n.03"},{"index":4,"sense_key":"class%1:04:00::","synset_id":"course.n.01"},{"index":5,"sense_key":"class%1:14:05::","synset_id":"class.n.05"},{"index":6,"sense_key":"class%1:14:04::","synset_id":"class.n.06"},{"index":7,"sense_key":"class%1:14:02::","synset_id":"class.n.07"},{"index":8,"sense_key":"class%1:07:00::","synset_id":"class.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"class%1:14:00::","call_ids":["senseval2.d001.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"class%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002695,"input_uncached_usd":0.0000323,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.176366625004448}
{"item_id":"senseval2.d001.s008.t005","gold_sense_keys":["simply%4:02:01::"],"candidates":[{"index":1,"sense_key":"simply%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"simply%4:02:03::","synset_id":"just.r.04"},{"index":3,"sense_key":"simply%4:02:02::","synset_id":"simply.r.03"},{"index":4,"sense_key":"simply%4:02:01::","synset_id":"plainly.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simply%4:02:00::","call_ids":["senseval2.d001.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simply%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028595,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.9675427920301445}
{"item_id":"senseval2.d001.s009.t000","gold_sense_keys":["function%2:35:00::"],"candidates":[{"index":1,"sense_key":"function%2:35:00::","synset_id":"function.v.01"},{"index":2,"sense_key":"function%2:42:12::","synset_id":"serve.v.01"},{"index":3,"sense_key":"function%2:33:00::","synset_id":"officiate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%2:35:00::","call_ids":["senseval2.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023539999999999998,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.821960500034038}
{"item_id":"senseval2.d001.s009.t004","gold_sense_keys":["hold%2:41:15::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":27,"chosen_sense_key":"hold%2:41:15::","call_ids":["senseval2.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":27,"predicted_sense_key":"hold%2:41:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1968,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00033519999999999996,"input_uncached_usd":0.00009839999999999999,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.373933124996256}
{"item_id":"senseval2.d001.s009.t005","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024189999999999997,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8407272080075927}
{"item_id":"senseval2.d001.s009.t006","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021729999999999997,"input_uncached_usd":0.0000317,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.777112834039144}
{"item_id":"senseval2.d001.s010.t001","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%2:30:01::","call_ids":["senseval2.d001.s010.t001__v1__a1","senseval2.d001.s010.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":2144,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00090025,"input_uncached_usd":0.00004264999999999999,"input_cached_usd":0.0,"output_usd":0.0008576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.192098792060278}
{"item_id":"senseval2.d001.s010.t003","gold_sense_keys":["radiation%1:19:00::"],"candidates":[{"index":1,"sense_key":"radiation%1:19:00::","synset_id":"radiation.n.01"},{"index":2,"sense_key":"radiation%1:04:00::","synset_id":"radiation.n.02"},{"index":3,"sense_key":"radiation%1:26:00::","synset_id":"radiation_sickness.n.01"},{"index":4,"sense_key":"radiation%1:22:00::","synset_id":"radiation.n.04"},{"index":5,"sense_key":"radiation%1:11:00::","synset_id":"radiation.n.05"},{"index":6,"sense_key":"radiation%1:08:00::","synset_id":"radiation.n.06"},{"index":7,"sense_key":"radiation%1:04:01::","synset_id":"radiotherapy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"radiation%1:19:00::","call_ids":["senseval2.d001.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"radiation%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003451,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.6500590419746}
{"item_id":"senseval2.d001.s010.t011","gold_sense_keys":["work%2:35:02::","work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"work%2:35:02::","call_ids":["senseval2.d001.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"work%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1505,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006448500000000001,"input_uncached_usd":0.00007525,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.18825674999971}
{"item_id":"senseval2.d001.s010.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s010.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001902,"input_uncached_usd":0.0000302,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3088252500165254}
{"item_id":"senseval2.d001.s010.t014","gold_sense_keys":["turn%2:30:03::","turn%2:42:00::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"turn%2:42:00::","call_ids":["senseval2.d001.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"turn%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1455,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00018195,"input_uncached_usd":0.00007274999999999999,"input_cached_usd":0.0,"output_usd":0.0001092,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3177944169729017}
{"item_id":"senseval2.d001.s011.t001","gold_sense_keys":["identify%2:31:01::"],"candidates":[{"index":1,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":2,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":3,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":4,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"},{"index":5,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":6,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"identify%2:31:01::","call_ids":["senseval2.d001.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"identify%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003692,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.234937957953662}
{"item_id":"senseval2.d001.s011.t003","gold_sense_keys":["differ%2:42:00::"],"candidates":[{"index":1,"sense_key":"differ%2:42:00::","synset_id":"differ.v.01"},{"index":2,"sense_key":"differ%2:32:00::","synset_id":"disagree.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"differ%2:42:00::","call_ids":["senseval2.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"differ%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006159000000000001,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.327638041984756}
{"item_id":"senseval2.d001.s011.t004","gold_sense_keys":["family%1:14:04::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"family%1:14:03::","call_ids":["senseval2.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"family%1:14:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029755,"input_uncached_usd":0.00003515,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.11756487499224}
{"item_id":"senseval2.d001.s011.t006","gold_sense_keys":["discover%2:31:00::","discover%2:31:03::","discover%2:36:00::","discover%2:39:03::"],"candidates":[{"index":1,"sense_key":"discover%2:39:03::","synset_id":"detect.v.01"},{"index":2,"sense_key":"discover%2:31:01::","synset_id":"learn.v.02"},{"index":3,"sense_key":"discover%2:36:00::","synset_id":"discover.v.03"},{"index":4,"sense_key":"discover%2:31:00::","synset_id":"discover.v.04"},{"index":5,"sense_key":"discover%2:40:00::","synset_id":"fall_upon.v.01"},{"index":6,"sense_key":"discover%2:32:04::","synset_id":"unwrap.v.02"},{"index":7,"sense_key":"discover%2:39:00::","synset_id":"discover.v.07"},{"index":8,"sense_key":"discover%2:31:03::","synset_id":"identify.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discover%2:36:00::","call_ids":["senseval2.d001.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discover%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055685,"input_uncached_usd":0.00003845,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.004083583015017}
{"item_id":"senseval2.d001.s011.t007","gold_sense_keys":["early%3:00:00::"],"candidates":[{"index":1,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":2,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":3,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":5,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":6,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"early%3:00:00::","call_ids":["senseval2.d001.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"early%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049755,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.03884804196423}
{"item_id":"senseval2.d001.s011.t008","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"call%2:32:01::","call_ids":["senseval2.d001.s011.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"call%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1513,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.00102925,"input_uncached_usd":0.00007565,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.5720305829891}
{"item_id":"senseval2.d001.s012.t001","gold_sense_keys":["present%3:00:02::"],"candidates":[{"index":1,"sense_key":"present%3:00:01::","synset_id":"present.a.01"},{"index":2,"sense_key":"present%3:00:02::","synset_id":"present.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"present%3:00:02::","call_ids":["senseval2.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"present%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051475,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.741520959010813}
{"item_id":"senseval2.d001.s012.t002","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000217,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.410922417009715}
{"item_id":"senseval2.d001.s012.t003","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"become%2:42:01::","call_ids":["senseval2.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"become%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004164,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.796843583986629}
{"item_id":"senseval2.d001.s012.t006","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1147,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047335,"input_uncached_usd":0.00005735,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.506274415994994}
{"item_id":"senseval2.d001.s012.t007","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"normal%3:00:03::","call_ids":["senseval2.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"normal%3:00:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023579999999999999,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.2499822080135345}
{"item_id":"senseval2.d001.s012.t008","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028474999999999996,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.445019583974499}
{"item_id":"senseval2.d001.s012.t009","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016575,"input_uncached_usd":0.000031349999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.784085542021785}
{"item_id":"senseval2.d001.s012.t010","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval2.d001.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016165,"input_uncached_usd":0.000027249999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6974325829651207}
{"item_id":"senseval2.d001.s012.t012","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["senseval2.d001.s012.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005416500000000001,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.263280707993545}
{"item_id":"senseval2.d001.s012.t013","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s012.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011099999999999999,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.076278207998257}
{"item_id":"senseval2.d001.s013.t000","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006675,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.906587750010658}
{"item_id":"senseval2.d001.s013.t001","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012984999999999998,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.860838667023927}
{"item_id":"senseval2.d001.s013.t003","gold_sense_keys":["come%2:30:03::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"come%2:30:03::","call_ids":["senseval2.d001.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"come%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1219,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005537500000000001,"input_uncached_usd":0.00006095,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.455386334040668}
{"item_id":"senseval2.d001.s013.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011054999999999999,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.8745149170281366}
{"item_id":"senseval2.d001.s013.t011","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"turn%2:42:00::","call_ids":["senseval2.d001.s013.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"turn%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1470,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00025909999999999996,"input_uncached_usd":0.0000735,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.98464120895369}
{"item_id":"senseval2.d001.s013.t013","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s013.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024214999999999998,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.756269166013226}
{"item_id":"senseval2.d001.s013.t015","gold_sense_keys":["only%4:02:03::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d001.s013.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005497,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.389537165989168}
{"item_id":"senseval2.d001.s013.t017","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%2:30:00::","call_ids":["senseval2.d001.s013.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041125,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.47156899998663}
{"item_id":"senseval2.d001.s014.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010584999999999998,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4970151250017807}
{"item_id":"senseval2.d001.s014.t003","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002362,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6119535829639062}
{"item_id":"senseval2.d001.s014.t004","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038615,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.287772667012177}
{"item_id":"senseval2.d001.s015.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002889,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3929856250179}
{"item_id":"senseval2.d001.s015.t003","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"need%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005693,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.707907792006154}
{"item_id":"senseval2.d001.s015.t004","gold_sense_keys":["control%2:41:01::"],"candidates":[{"index":1,"sense_key":"control%2:41:00::","synset_id":"control.v.01"},{"index":2,"sense_key":"control%2:41:01::","synset_id":"control.v.02"},{"index":3,"sense_key":"control%2:35:00::","synset_id":"operate.v.03"},{"index":4,"sense_key":"control%2:37:06::","synset_id":"manipulate.v.05"},{"index":5,"sense_key":"control%2:41:02::","synset_id":"control.v.05"},{"index":6,"sense_key":"control%2:31:11::","synset_id":"control.v.06"},{"index":7,"sense_key":"control%2:31:01::","synset_id":"see.v.10"},{"index":8,"sense_key":"control%2:31:00::","synset_id":"master.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"control%2:41:01::","call_ids":["senseval2.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"control%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042875,"input_uncached_usd":0.00003835,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.936113667034078}
{"item_id":"senseval2.d001.s015.t005","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029505,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.779075041005854}
{"item_id":"senseval2.d001.s015.t006","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003472,"input_uncached_usd":0.0000336,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3477305410197005}
{"item_id":"senseval2.d001.s015.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011224999999999999,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2728173339855857}
{"item_id":"senseval2.d001.s015.t008","gold_sense_keys":["arise%2:42:00::"],"candidates":[{"index":1,"sense_key":"arise%2:42:00::","synset_id":"originate.v.01"},{"index":2,"sense_key":"arise%2:42:01::","synset_id":"arise.v.02"},{"index":3,"sense_key":"arise%2:38:00::","synset_id":"arise.v.03"},{"index":4,"sense_key":"arise%2:30:00::","synset_id":"arise.v.04"},{"index":5,"sense_key":"arise%2:38:04::","synset_id":"rise.v.01"},{"index":6,"sense_key":"arise%2:41:00::","synset_id":"rebel.v.01"},{"index":7,"sense_key":"arise%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"arise%2:30:00::","call_ids":["senseval2.d001.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"arise%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037429999999999994,"input_uncached_usd":0.0000351,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4409894160344265}
{"item_id":"senseval2.d001.s015.t009","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046809999999999994,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.101838542032056}
{"item_id":"senseval2.d001.s015.t010","gold_sense_keys":["impaired%3:00:00::"],"candidates":[{"index":1,"sense_key":"impaired%3:00:00::","synset_id":"impaired.a.01"},{"index":2,"sense_key":"impaired%5:00:00:unfit:01","synset_id":"afflicted.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impaired%3:00:00::","call_ids":["senseval2.d001.s015.t010__v1__a1","senseval2.d001.s015.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impaired%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041815,"input_uncached_usd":0.00004695,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.250937959004659}
{"item_id":"senseval2.d001.s016.t000","gold_sense_keys":["person%1:03:00::"],"candidates":[{"index":1,"sense_key":"person%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"person%1:08:00::","synset_id":"person.n.02"},{"index":3,"sense_key":"person%1:10:00::","synset_id":"person.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"person%1:03:00::","call_ids":["senseval2.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"person%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002628,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.387503750040196}
{"item_id":"senseval2.d001.s016.t001","gold_sense_keys":["bear%2:29:01::"],"candidates":[{"index":1,"sense_key":"bear%2:42:01::","synset_id":"bear.v.01"},{"index":2,"sense_key":"bear%2:29:01::","synset_id":"give_birth.v.01"},{"index":3,"sense_key":"bear%2:31:00::","synset_id":"digest.v.03"},{"index":4,"sense_key":"bear%2:35:00::","synset_id":"bear.v.04"},{"index":5,"sense_key":"bear%2:36:00::","synset_id":"bear.v.05"},{"index":6,"sense_key":"bear%2:40:02::","synset_id":"bear.v.06"},{"index":7,"sense_key":"bear%2:42:14::","synset_id":"hold.v.11"},{"index":8,"sense_key":"bear%2:40:01::","synset_id":"yield.v.10"},{"index":9,"sense_key":"bear%2:29:02::","synset_id":"wear.v.02"},{"index":10,"sense_key":"bear%2:41:00::","synset_id":"behave.v.02"},{"index":11,"sense_key":"bear%2:40:00::","synset_id":"bear.v.11"},{"index":12,"sense_key":"bear%2:35:01::","synset_id":"hold.v.14"},{"index":13,"sense_key":"bear%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bear%2:42:01::","call_ids":["senseval2.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bear%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":2512,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.00105225,"input_uncached_usd":0.000047449999999999995,"input_cached_usd":0.0,"output_usd":0.0010048,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.143433000019286}
{"item_id":"senseval2.d001.s016.t003","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003141,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.726909250020981}
{"item_id":"senseval2.d001.s016.t005","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004933,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.369795957987662}
{"item_id":"senseval2.d001.s016.t006","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%2:30:01::","call_ids":["senseval2.d001.s016.t006__v1__a1","senseval2.d001.s016.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":2144,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00090435,"input_uncached_usd":0.00004675,"input_cached_usd":0.0,"output_usd":0.0008576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.929068666067906}
{"item_id":"senseval2.d001.s016.t007","gold_sense_keys":["early%4:02:01::"],"candidates":[{"index":1,"sense_key":"early%4:02:01::","synset_id":"early_on.r.01"},{"index":2,"sense_key":"early%4:02:00::","synset_id":"early.r.02"},{"index":3,"sense_key":"early%4:02:02::","synset_id":"early.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"early%4:02:01::","call_ids":["senseval2.d001.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"early%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015879999999999998,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.628313499968499}
{"item_id":"senseval2.d001.s016.t008","gold_sense_keys":["life%1:28:00::","life%1:28:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"life%1:28:01::","call_ids":["senseval2.d001.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"life%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030934999999999996,"input_uncached_usd":0.000046949999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.009505332971457}
{"item_id":"senseval2.d001.s016.t009","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036194999999999993,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.587227042007726}
{"item_id":"senseval2.d001.s016.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011184999999999998,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9848540420061909}
{"item_id":"senseval2.d001.s016.t012","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039779999999999997,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.331160125031602}
{"item_id":"senseval2.d001.s016.t013","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s016.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022784999999999998,"input_uncached_usd":0.00004185,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4723887080326676}
{"item_id":"senseval2.d001.s016.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s016.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001873,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2560742080095224}
{"item_id":"senseval2.d001.s016.t015","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s016.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001605,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.967467666021548}
{"item_id":"senseval2.d001.s016.t016","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s016.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018864999999999998,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8353353340062313}
{"item_id":"senseval2.d001.s016.t017","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s016.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1312,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007632,"input_uncached_usd":0.0000656,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.342733124969527}
{"item_id":"senseval2.d001.s017.t000","gold_sense_keys":["emerging%5:00:00:nascent:00"],"candidates":[{"index":1,"sense_key":"emerging%5:00:00:future:00","synset_id":"emerging.s.01"},{"index":2,"sense_key":"emerging%5:00:00:nascent:00","synset_id":"emergent.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emerging%5:00:00:nascent:00","call_ids":["senseval2.d001.s017.t000__v1__a1","senseval2.d001.s017.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emerging%5:00:00:nascent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00059645,"input_uncached_usd":0.000046049999999999994,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.387517791998107}
{"item_id":"senseval2.d001.s017.t001","gold_sense_keys":["genetic%3:01:00::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031515,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.919262332958169}
{"item_id":"senseval2.d001.s017.t003","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval2.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021314999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6922402920317836}
{"item_id":"senseval2.d001.s017.t006","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010449999999999999,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.8500858329935}
{"item_id":"senseval2.d001.s017.t009","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval2.d001.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.000394,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.826767250022385}
{"item_id":"senseval2.d001.s017.t010","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":848,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00074,"input_uncached_usd":0.0000424,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.711864999961108}
{"item_id":"senseval2.d001.s017.t011","gold_sense_keys":["age%1:28:02::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"age%1:28:02::","call_ids":["senseval2.d001.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"age%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026605,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.112193875014782}
{"item_id":"senseval2.d001.s017.t013","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s017.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029065,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.52721329202177}
{"item_id":"senseval2.d001.s018.t000","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d001.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031975,"input_uncached_usd":0.00003175,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.615736124978866}
{"item_id":"senseval2.d001.s018.t001","gold_sense_keys":["beneficiary%1:18:00::"],"candidates":[{"index":1,"sense_key":"beneficiary%1:18:00::","synset_id":"beneficiary.n.01"},{"index":2,"sense_key":"beneficiary%1:10:00::","synset_id":"benefactive_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"beneficiary%1:18:00::","call_ids":["senseval2.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"beneficiary%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046419999999999995,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.37635837495327}
{"item_id":"senseval2.d001.s018.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022859999999999997,"input_uncached_usd":0.000042999999999999995,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7168290420086123}
{"item_id":"senseval2.d001.s018.t003","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"},{"index":2,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":3,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finding%1:17:00::","call_ids":["senseval2.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026145,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.249892375024501}
{"item_id":"senseval2.d001.s019.t000","gold_sense_keys":["couple%1:14:01::"],"candidates":[{"index":1,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":2,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":5,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"couple%1:14:01::","call_ids":["senseval2.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"couple%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036754999999999996,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.305968375003431}
{"item_id":"senseval2.d001.s019.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00051215,"input_uncached_usd":0.000044949999999999995,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.2868127500405535}
{"item_id":"senseval2.d001.s019.t002","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d001.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044385,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.310641125019174}
{"item_id":"senseval2.d001.s019.t003","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"before%4:02:03::","call_ids":["senseval2.d001.s019.t003__v1__a1","senseval2.d001.s019.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":736,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00034384999999999993,"input_uncached_usd":0.000049449999999999996,"input_cached_usd":0.0,"output_usd":0.0002944,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.088730291055981}
{"item_id":"senseval2.d001.s019.t004","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021299999999999997,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.67114887497155}
{"item_id":"senseval2.d001.s019.t005","gold_sense_keys":["pregnant%3:00:00::"],"candidates":[{"index":1,"sense_key":"pregnant%3:00:00::","synset_id":"pregnant.a.01"},{"index":2,"sense_key":"pregnant%5:00:00:meaningful:00","synset_id":"meaning.s.01"},{"index":3,"sense_key":"pregnant%5:00:00:full:00","synset_id":"fraught.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pregnant%3:00:00::","call_ids":["senseval2.d001.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pregnant%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001342,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0001092,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.202113042003475}
{"item_id":"senseval2.d001.s019.t006","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d001.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016265,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4773796669906005}
{"item_id":"senseval2.d001.s019.t007","gold_sense_keys":["chance%1:07:00::"],"candidates":[{"index":1,"sense_key":"chance%1:26:00::","synset_id":"opportunity.n.01"},{"index":2,"sense_key":"chance%1:19:00::","synset_id":"luck.n.02"},{"index":3,"sense_key":"chance%1:04:00::","synset_id":"chance.n.03"},{"index":4,"sense_key":"chance%1:07:00::","synset_id":"probability.n.01"},{"index":5,"sense_key":"chance%1:26:01::","synset_id":"prospect.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"chance%1:07:00::","call_ids":["senseval2.d001.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"chance%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004217,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.686845624994021}
{"item_id":"senseval2.d001.s019.t010","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036819999999999995,"input_uncached_usd":0.000029,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.026707459008321}
{"item_id":"senseval2.d001.s019.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001124,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1021885839872994}
{"item_id":"senseval2.d001.s019.t012","gold_sense_keys":["occur%2:42:03::"],"candidates":[{"index":1,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"},{"index":3,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"occur%2:30:00::","call_ids":["senseval2.d001.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"occur%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046794999999999996,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.712810750002973}
{"item_id":"senseval2.d001.s019.t013","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005537000000000001,"input_uncached_usd":0.0000353,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.429156499973033}
{"item_id":"senseval2.d001.s019.t014","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:00::","call_ids":["senseval2.d001.s019.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028734999999999996,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.566125958983321}
{"item_id":"senseval2.d001.s019.t015","gold_sense_keys":["birth%1:18:00::"],"candidates":[{"index":1,"sense_key":"birth%1:28:00::","synset_id":"birth.n.01"},{"index":2,"sense_key":"birth%1:11:00::","synset_id":"birth.n.02"},{"index":3,"sense_key":"birth%1:22:00::","synset_id":"parturition.n.01"},{"index":4,"sense_key":"birth%1:24:00::","synset_id":"parentage.n.02"},{"index":5,"sense_key":"birth%1:18:00::","synset_id":"birth.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"birth%1:11:00::","call_ids":["senseval2.d001.s019.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"birth%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007011,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.301843790977728}
{"item_id":"senseval2.d001.s020.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval2.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00033355,"input_uncached_usd":0.000045549999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.642704083991703}
{"item_id":"senseval2.d001.s020.t001","gold_sense_keys":["carry%2:42:13::"],"candidates":[{"index":1,"sense_key":"carry%2:35:02::","synset_id":"transport.v.02"},{"index":2,"sense_key":"carry%2:42:01::","synset_id":"carry.v.02"},{"index":3,"sense_key":"carry%2:38:03::","synset_id":"impart.v.03"},{"index":4,"sense_key":"carry%2:32:02::","synset_id":"carry.v.04"},{"index":5,"sense_key":"carry%2:35:09::","synset_id":"carry.v.05"},{"index":6,"sense_key":"carry%2:35:01::","synset_id":"hold.v.14"},{"index":7,"sense_key":"carry%2:42:03::","synset_id":"hold.v.11"},{"index":8,"sense_key":"carry%2:41:00::","synset_id":"carry.v.08"},{"index":9,"sense_key":"carry%2:30:05::","synset_id":"carry.v.09"},{"index":10,"sense_key":"carry%2:42:02::","synset_id":"carry.v.10"},{"index":11,"sense_key":"carry%2:33:01::","synset_id":"carry.v.11"},{"index":12,"sense_key":"carry%2:31:12::","synset_id":"carry.v.12"},{"index":13,"sense_key":"carry%2:41:01::","synset_id":"behave.v.02"},{"index":14,"sense_key":"carry%2:40:00::","synset_id":"stock.v.01"},{"index":15,"sense_key":"carry%2:36:00::","synset_id":"carry.v.15"},{"index":16,"sense_key":"carry%2:35:08::","synset_id":"dribble.v.03"},{"index":17,"sense_key":"carry%2:32:07::","synset_id":"carry.v.17"},{"index":18,"sense_key":"carry%2:42:13::","synset_id":"carry.v.18"},{"index":19,"sense_key":"carry%2:42:15::","synset_id":"carry.v.19"},{"index":20,"sense_key":"carry%2:42:14::","synset_id":"carry.v.20"},{"index":21,"sense_key":"carry%2:42:12::","synset_id":"carry.v.21"},{"index":22,"sense_key":"carry%2:42:08::","synset_id":"carry.v.22"},{"index":23,"sense_key":"carry%2:41:02::","synset_id":"carry.v.23"},{"index":24,"sense_key":"carry%2:41:06::","synset_id":"carry.v.24"},{"index":25,"sense_key":"carry%2:41:12::","synset_id":"carry.v.25"},{"index":26,"sense_key":"carry%2:40:10::","synset_id":"carry.v.26"},{"index":27,"sense_key":"carry%2:40:04::","synset_id":"carry.v.27"},{"index":28,"sense_key":"carry%2:40:13::","synset_id":"post.v.07"},{"index":29,"sense_key":"carry%2:38:10::","synset_id":"carry.v.29"},{"index":30,"sense_key":"carry%2:38:07::","synset_id":"carry.v.30"},{"index":31,"sense_key":"carry%2:36:08::","synset_id":"carry.v.31"},{"index":32,"sense_key":"carry%2:35:12::","synset_id":"carry.v.32"},{"index":33,"sense_key":"carry%2:34:00::","synset_id":"carry.v.33"},{"index":34,"sense_key":"carry%2:34:01::","synset_id":"carry.v.34"},{"index":35,"sense_key":"carry%2:33:04::","synset_id":"carry.v.35"},{"index":36,"sense_key":"carry%2:33:03::","synset_id":"carry.v.36"},{"index":37,"sense_key":"carry%2:33:02::","synset_id":"carry.v.37"},{"index":38,"sense_key":"carry%2:33:12::","synset_id":"carry.v.38"},{"index":39,"sense_key":"carry%2:32:00::","synset_id":"carry.v.39"},{"index":40,"sense_key":"carry%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":21,"chosen_sense_key":"carry%2:42:12::","call_ids":["senseval2.d001.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":21,"predicted_sense_key":"carry%2:42:12::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1985,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00054125,"input_uncached_usd":0.00009925,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.8621812500059605}
{"item_id":"senseval2.d001.s020.t002","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015854999999999998,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.923042291018646}
{"item_id":"senseval2.d001.s020.t004","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005614,"input_uncached_usd":0.000042999999999999995,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.348733208025806}
{"item_id":"senseval2.d001.s020.t005","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018959999999999997,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1300118750077672}
{"item_id":"senseval2.d001.s020.t008","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029614999999999996,"input_uncached_usd":0.00003375,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.728265792015009}
{"item_id":"senseval2.d001.s020.t009","gold_sense_keys":["mother%1:18:00::"],"candidates":[{"index":1,"sense_key":"mother%1:18:00::","synset_id":"mother.n.01"},{"index":2,"sense_key":"mother%1:27:00::","synset_id":"mother.n.02"},{"index":3,"sense_key":"mother%1:18:02::","synset_id":"mother.n.03"},{"index":4,"sense_key":"mother%1:18:01::","synset_id":"mother.n.04"},{"index":5,"sense_key":"mother%1:09:00::","synset_id":"mother.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mother%1:18:00::","call_ids":["senseval2.d001.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mother%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023979999999999997,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.217815542011522}
{"item_id":"senseval2.d001.s020.t010","gold_sense_keys":["suffer%2:39:01::","suffer%2:39:02::"],"candidates":[{"index":1,"sense_key":"suffer%2:39:01::","synset_id":"suffer.v.01"},{"index":2,"sense_key":"suffer%2:29:01::","synset_id":"suffer.v.02"},{"index":3,"sense_key":"suffer%2:37:00::","synset_id":"suffer.v.03"},{"index":4,"sense_key":"suffer%2:31:00::","synset_id":"digest.v.03"},{"index":5,"sense_key":"suffer%2:30:00::","synset_id":"suffer.v.05"},{"index":6,"sense_key":"suffer%2:29:00::","synset_id":"suffer.v.06"},{"index":7,"sense_key":"suffer%2:39:00::","synset_id":"hurt.v.06"},{"index":8,"sense_key":"suffer%2:29:03::","synset_id":"suffer.v.08"},{"index":9,"sense_key":"suffer%2:42:04::","synset_id":"suffer.v.09"},{"index":10,"sense_key":"suffer%2:39:02::","synset_id":"suffer.v.10"},{"index":11,"sense_key":"suffer%2:30:02::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"suffer%2:39:02::","call_ids":["senseval2.d001.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"suffer%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009428500000000001,"input_uncached_usd":0.00004045,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.740801415988244}
{"item_id":"senseval2.d001.s020.t011","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval2.d001.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":1873,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007770499999999998,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.0007492,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.482632667000871}
{"item_id":"senseval2.d001.s020.t012","gold_sense_keys":["fate%1:26:00::"],"candidates":[{"index":1,"sense_key":"fate%1:11:00::","synset_id":"destiny.n.01"},{"index":2,"sense_key":"fate%1:18:00::","synset_id":"destiny.n.02"},{"index":3,"sense_key":"fate%1:26:00::","synset_id":"fortune.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fate%1:26:00::","call_ids":["senseval2.d001.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003925,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.159839583968278}
{"item_id":"senseval2.d001.s020.t013","gold_sense_keys":["baby%1:18:00::"],"candidates":[{"index":1,"sense_key":"baby%1:18:00::","synset_id":"baby.n.01"},{"index":2,"sense_key":"baby%1:18:01::","synset_id":"baby.n.02"},{"index":3,"sense_key":"baby%1:18:03::","synset_id":"child.n.03"},{"index":4,"sense_key":"baby%1:18:04::","synset_id":"baby.n.04"},{"index":5,"sense_key":"baby%1:18:02::","synset_id":"baby.n.05"},{"index":6,"sense_key":"baby%1:05:00::","synset_id":"baby.n.06"},{"index":7,"sense_key":"baby%1:04:00::","synset_id":"baby.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"baby%1:18:04::","call_ids":["senseval2.d001.s020.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"baby%1:18:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":3153,"reasoning_output_tokens":3136},"cost":{"currency":"USD","total_usd":0.00129525,"input_uncached_usd":0.00003405,"input_cached_usd":0.0,"output_usd":0.0012611999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":24.77070629101945}
{"item_id":"senseval2.d001.s021.t000","gold_sense_keys":["isolation%1:04:00::"],"candidates":[{"index":1,"sense_key":"isolation%1:26:00::","synset_id":"isolation.n.01"},{"index":2,"sense_key":"isolation%1:12:00::","synset_id":"isolation.n.02"},{"index":3,"sense_key":"isolation%1:04:00::","synset_id":"isolation.n.03"},{"index":4,"sense_key":"isolation%1:22:00::","synset_id":"isolation.n.04"},{"index":5,"sense_key":"isolation%1:04:01::","synset_id":"isolation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"isolation%1:04:00::","call_ids":["senseval2.d001.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"isolation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018864999999999998,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0903992910170928}
{"item_id":"senseval2.d001.s021.t003","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d001.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029144999999999996,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.009415250038728}
{"item_id":"senseval2.d001.s021.t004","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041475,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.081036625022534}
{"item_id":"senseval2.d001.s021.t005","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006878499999999999,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.024730250006542}
{"item_id":"senseval2.d001.s021.t007","gold_sense_keys":["find_out%2:32:00::"],"candidates":[{"index":1,"sense_key":"find_out%2:32:00::","synset_id":"determine.v.01"},{"index":2,"sense_key":"find_out%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"find_out%2:32:01::","synset_id":"determine.v.08"},{"index":4,"sense_key":"find_out%2:31:01::","synset_id":"find_out.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"find_out%2:32:01::","call_ids":["senseval2.d001.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"find_out%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026995,"input_uncached_usd":0.00003315,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0479527080315165}
{"item_id":"senseval2.d001.s021.t008","gold_sense_keys":["threat%1:26:00::"],"candidates":[{"index":1,"sense_key":"threat%1:26:00::","synset_id":"menace.n.01"},{"index":2,"sense_key":"threat%1:10:00::","synset_id":"threat.n.02"},{"index":3,"sense_key":"threat%1:10:01::","synset_id":"threat.n.03"},{"index":4,"sense_key":"threat%1:18:00::","synset_id":"terror.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"threat%1:26:00::","call_ids":["senseval2.d001.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"threat%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003424,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.341752791951876}
{"item_id":"senseval2.d001.s021.t009","gold_sense_keys":["baby%1:18:00::"],"candidates":[{"index":1,"sense_key":"baby%1:18:00::","synset_id":"baby.n.01"},{"index":2,"sense_key":"baby%1:18:01::","synset_id":"baby.n.02"},{"index":3,"sense_key":"baby%1:18:03::","synset_id":"child.n.03"},{"index":4,"sense_key":"baby%1:18:04::","synset_id":"baby.n.04"},{"index":5,"sense_key":"baby%1:18:02::","synset_id":"baby.n.05"},{"index":6,"sense_key":"baby%1:05:00::","synset_id":"baby.n.06"},{"index":7,"sense_key":"baby%1:04:00::","synset_id":"baby.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"baby%1:18:00::","call_ids":["senseval2.d001.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"baby%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032315,"input_uncached_usd":0.00003515,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.636171624995768}
{"item_id":"senseval2.d001.s021.t010","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":2,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"},{"index":3,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":4,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":5,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":6,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":7,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":8,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":9,"sense_key":"face%2:35:01::","synset_id":"face.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval2.d001.s021.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006867499999999999,"input_uncached_usd":0.000039949999999999995,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.88131120800972}
{"item_id":"senseval2.d001.s022.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024465,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.026263458014}
{"item_id":"senseval2.d001.s022.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056195,"input_uncached_usd":0.000043549999999999995,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.406599749985617}
{"item_id":"senseval2.d001.s022.t003","gold_sense_keys":["genetic%3:01:00::","genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003166,"input_uncached_usd":0.000028199999999999998,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.005997625004966}
{"item_id":"senseval2.d001.s022.t004","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%2:32:00::","call_ids":["senseval2.d001.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":956,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041259999999999995,"input_uncached_usd":0.000047799999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.932613290962763}
{"item_id":"senseval2.d001.s022.t005","gold_sense_keys":["little%5:00:00:emotional:00","little%5:00:00:young:00"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"little%5:00:00:young:00","call_ids":["senseval2.d001.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"little%5:00:00:young:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022419999999999997,"input_uncached_usd":0.000038599999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.630527417000849}
{"item_id":"senseval2.d001.s022.t006","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.000056449999999999997,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.000032,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.2519914580043405}
{"item_id":"senseval2.d001.s022.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031175,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.325249208020978}
{"item_id":"senseval2.d001.s022.t011","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":2,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"},{"index":3,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":4,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":5,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":6,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":7,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":8,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":9,"sense_key":"face%2:35:01::","synset_id":"face.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval2.d001.s022.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048014999999999993,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.287632625026163}
{"item_id":"senseval2.d001.s022.t012","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d001.s022.t012__v1__a1","senseval2.d001.s022.t012__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1023,"cached_input_tokens":0,"output_tokens":2080,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008831500000000001,"input_uncached_usd":0.000051149999999999996,"input_cached_usd":0.0,"output_usd":0.000832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.435113749001175}
{"item_id":"senseval2.d001.s022.t013","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["senseval2.d001.s022.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036754999999999996,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.906089209020138}
{"item_id":"senseval2.d001.s022.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s022.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034165,"input_uncached_usd":0.000028049999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1643872919958085}
{"item_id":"senseval2.d001.s022.t015","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d001.s022.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031605,"input_uncached_usd":0.000028049999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.581622791010886}
{"item_id":"senseval2.d001.s022.t016","gold_sense_keys":["develop%2:29:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s022.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1329,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00035445,"input_uncached_usd":0.00006645,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.534555250022095}
{"item_id":"senseval2.d001.s022.t018","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s022.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044534999999999996,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.171602749964222}
{"item_id":"senseval2.d001.s023.t000","gold_sense_keys":["make%2:30:10::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":45,"chosen_sense_key":"make%2:30:10::","call_ids":["senseval2.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":45,"predicted_sense_key":"make%2:30:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2328,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004812,"input_uncached_usd":0.0001164,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.661079332989175}
{"item_id":"senseval2.d001.s023.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005101999999999999,"input_uncached_usd":0.0000426,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.232372500002384}
{"item_id":"senseval2.d001.s024.t001","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d001.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039769999999999996,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9733643750078045}
{"item_id":"senseval2.d001.s024.t004","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["senseval2.d001.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018304999999999998,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.845474333036691}
{"item_id":"senseval2.d001.s024.t005","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval2.d001.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039779999999999997,"input_uncached_usd":0.000032999999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.167506915982813}
{"item_id":"senseval2.d001.s024.t006","gold_sense_keys":["develop%2:29:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1340,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005598000000000001,"input_uncached_usd":0.000067,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.719319667026866}
{"item_id":"senseval2.d001.s024.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016434999999999998,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8577736250008456}
{"item_id":"senseval2.d001.s025.t000","gold_sense_keys":["equally%4:02:01::"],"candidates":[{"index":1,"sense_key":"equally%4:02:01::","synset_id":"equally.r.01"},{"index":2,"sense_key":"equally%4:02:02::","synset_id":"evenly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equally%4:02:01::","call_ids":["senseval2.d001.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004149,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.279319749970455}
{"item_id":"senseval2.d001.s025.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["senseval2.d001.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039525,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.983146583021153}
{"item_id":"senseval2.d001.s025.t003","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00085135,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.44784175005043}
{"item_id":"senseval2.d001.s025.t005","gold_sense_keys":["control%2:41:01::"],"candidates":[{"index":1,"sense_key":"control%2:41:00::","synset_id":"control.v.01"},{"index":2,"sense_key":"control%2:41:01::","synset_id":"control.v.02"},{"index":3,"sense_key":"control%2:35:00::","synset_id":"operate.v.03"},{"index":4,"sense_key":"control%2:37:06::","synset_id":"manipulate.v.05"},{"index":5,"sense_key":"control%2:41:02::","synset_id":"control.v.05"},{"index":6,"sense_key":"control%2:31:11::","synset_id":"control.v.06"},{"index":7,"sense_key":"control%2:31:01::","synset_id":"see.v.10"},{"index":8,"sense_key":"control%2:31:00::","synset_id":"master.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"control%2:41:00::","call_ids":["senseval2.d001.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"control%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.000685,"input_uncached_usd":0.000038599999999999996,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.829600624972954}
{"item_id":"senseval2.d001.s025.t007","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":338,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016810000000000002,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.0001352,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.762249709048774}
{"item_id":"senseval2.d001.s025.t008","gold_sense_keys":["growth%1:22:00::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["senseval2.d001.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00075705,"input_uncached_usd":0.000033849999999999996,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.201757791975979}
{"item_id":"senseval2.d001.s025.t009","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval2.d001.s025.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2342,"cached_input_tokens":0,"output_tokens":3408,"reasoning_output_tokens":3392},"cost":{"currency":"USD","total_usd":0.0014803,"input_uncached_usd":0.0001171,"input_cached_usd":0.0,"output_usd":0.0013632,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.048767125001177}
{"item_id":"senseval2.d001.s025.t010","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":3,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doctor%1:18:00::","call_ids":["senseval2.d001.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024099999999999998,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.219856999989133}
{"item_id":"senseval2.d001.s025.t011","gold_sense_keys":["name%2:32:03::"],"candidates":[{"index":1,"sense_key":"name%2:32:03::","synset_id":"name.v.01"},{"index":2,"sense_key":"name%2:32:02::","synset_id":"name.v.02"},{"index":3,"sense_key":"name%2:41:01::","synset_id":"name.v.03"},{"index":4,"sense_key":"name%2:41:00::","synset_id":"appoint.v.01"},{"index":5,"sense_key":"name%2:32:07::","synset_id":"name.v.05"},{"index":6,"sense_key":"name%2:32:04::","synset_id":"mention.v.01"},{"index":7,"sense_key":"name%2:31:01::","synset_id":"identify.v.05"},{"index":8,"sense_key":"name%2:32:09::","synset_id":"list.v.01"},{"index":9,"sense_key":"name%2:31:00::","synset_id":"diagnose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"name%2:32:07::","call_ids":["senseval2.d001.s025.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"name%2:32:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048029999999999997,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.029302915965673}
{"item_id":"senseval2.d001.s025.t012","gold_sense_keys":["open%2:30:00::"],"candidates":[{"index":1,"sense_key":"open%2:35:00::","synset_id":"open.v.01"},{"index":2,"sense_key":"open%2:41:00::","synset_id":"open.v.02"},{"index":3,"sense_key":"open%2:35:06::","synset_id":"open.v.03"},{"index":4,"sense_key":"open%2:41:01::","synset_id":"open.v.04"},{"index":5,"sense_key":"open%2:35:08::","synset_id":"unfold.v.04"},{"index":6,"sense_key":"open%2:30:00::","synset_id":"open.v.06"},{"index":7,"sense_key":"open%2:30:01::","synset_id":"open.v.07"},{"index":8,"sense_key":"open%2:42:00::","synset_id":"open.v.08"},{"index":9,"sense_key":"open%2:33:00::","synset_id":"open.v.09"},{"index":10,"sense_key":"open%2:40:04::","synset_id":"afford.v.04"},{"index":11,"sense_key":"open%2:39:10::","synset_id":"open.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"open%2:30:00::","call_ids":["senseval2.d001.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"open%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003521,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.874022082949523}
{"item_id":"senseval2.d001.s025.t014","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s025.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003433,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.581966124998871}
{"item_id":"senseval2.d001.s025.t016","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d001.s025.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025985,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.604208583012223}
{"item_id":"senseval2.d001.s025.t017","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s025.t017__v1__a1","senseval2.d001.s025.t017__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":2592,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.00108525,"input_uncached_usd":0.00004845,"input_cached_usd":0.0,"output_usd":0.0010368,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.776127292017918}
{"item_id":"senseval2.d001.s025.t018","gold_sense_keys":["explode%2:30:02::"],"candidates":[{"index":1,"sense_key":"explode%2:30:01::","synset_id":"explode.v.01"},{"index":2,"sense_key":"explode%2:30:00::","synset_id":"explode.v.02"},{"index":3,"sense_key":"explode%2:31:00::","synset_id":"explode.v.03"},{"index":4,"sense_key":"explode%2:30:03::","synset_id":"explode.v.04"},{"index":5,"sense_key":"explode%2:35:00::","synset_id":"explode.v.05"},{"index":6,"sense_key":"explode%2:32:02::","synset_id":"explode.v.06"},{"index":7,"sense_key":"explode%2:32:00::","synset_id":"explode.v.07"},{"index":8,"sense_key":"explode%2:31:01::","synset_id":"explode.v.08"},{"index":9,"sense_key":"explode%2:30:04::","synset_id":"detonate.v.02"},{"index":10,"sense_key":"explode%2:30:02::","synset_id":"explode.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"explode%2:30:02::","call_ids":["senseval2.d001.s025.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"explode%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002496,"input_uncached_usd":0.0000384,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.34730166697409}
{"item_id":"senseval2.d001.s026.t000","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":3,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":4,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":5,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":6,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":7,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":8,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":9,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":10,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":11,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":12,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn_out%2:42:01::","call_ids":["senseval2.d001.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn_out%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00023035,"input_uncached_usd":0.00004475,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5313040830078535}
{"item_id":"senseval2.d001.s026.t001","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["senseval2.d001.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021774999999999998,"input_uncached_usd":0.000032149999999999995,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.310972416016739}
{"item_id":"senseval2.d001.s026.t002","gold_sense_keys":["tragic%5:00:00:sad:00"],"candidates":[{"index":1,"sense_key":"tragic%5:00:00:sad:00","synset_id":"tragic.s.01"},{"index":2,"sense_key":"tragic%3:01:00::","synset_id":"tragic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tragic%5:00:00:sad:00","call_ids":["senseval2.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tragic%5:00:00:sad:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004896999999999999,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.517940542020369}
{"item_id":"senseval2.d001.s026.t003","gold_sense_keys":["uncommon%3:00:00::"],"candidates":[{"index":1,"sense_key":"uncommon%3:00:00::","synset_id":"uncommon.a.01"},{"index":2,"sense_key":"uncommon%5:00:00:extraordinary:00","synset_id":"rare.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uncommon%3:00:00::","call_ids":["senseval2.d001.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uncommon%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036344999999999997,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.12284358299803}
{"item_id":"senseval2.d001.s026.t005","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval2.d001.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2329,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00040445,"input_uncached_usd":0.00011645,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.822579499974381}
{"item_id":"senseval2.d001.s026.t006","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005665499999999999,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.189543082960881}
{"item_id":"senseval2.d001.s026.t008","gold_sense_keys":["insight%1:09:01::","insight%1:09:02::"],"candidates":[{"index":1,"sense_key":"insight%1:09:02::","synset_id":"penetration.n.02"},{"index":2,"sense_key":"insight%1:12:00::","synset_id":"insight.n.02"},{"index":3,"sense_key":"insight%1:09:01::","synset_id":"insight.n.03"},{"index":4,"sense_key":"insight%1:09:00::","synset_id":"insight.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"insight%1:09:01::","call_ids":["senseval2.d001.s026.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"insight%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026205,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0718762910109945}
{"item_id":"senseval2.d001.s026.t009","gold_sense_keys":["basic%3:00:00::"],"candidates":[{"index":1,"sense_key":"basic%3:00:00::","synset_id":"basic.a.01"},{"index":2,"sense_key":"basic%5:00:00:standard:01","synset_id":"basic.s.01"},{"index":3,"sense_key":"basic%5:00:02:first:00","synset_id":"basic.s.02"},{"index":4,"sense_key":"basic%5:00:00:alkaline:00","synset_id":"basic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"basic%3:00:00::","call_ids":["senseval2.d001.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"basic%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044304999999999995,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.210001375002321}
{"item_id":"senseval2.d001.s026.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s026.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021459999999999998,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.977781541994773}
{"item_id":"senseval2.d001.s026.t011","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s026.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030544999999999997,"input_uncached_usd":0.00004265,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.467319791961927}
{"item_id":"senseval2.d001.s026.t012","gold_sense_keys":["director%1:18:00::"],"candidates":[{"index":1,"sense_key":"director%1:18:00::","synset_id":"director.n.01"},{"index":2,"sense_key":"director%1:18:03::","synset_id":"director.n.02"},{"index":3,"sense_key":"director%1:18:02::","synset_id":"director.n.03"},{"index":4,"sense_key":"director%1:18:04::","synset_id":"film_director.n.01"},{"index":5,"sense_key":"director%1:18:01::","synset_id":"conductor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"director%1:18:00::","call_ids":["senseval2.d001.s026.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"director%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002889,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.434689458983485}
{"item_id":"senseval2.d001.s027.t001","gold_sense_keys":["public%1:14:00::"],"candidates":[{"index":1,"sense_key":"public%1:14:00::","synset_id":"populace.n.01"},{"index":2,"sense_key":"public%1:14:01::","synset_id":"public.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%1:14:00::","call_ids":["senseval2.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030985,"input_uncached_usd":0.00002145,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.504136500007007}
{"item_id":"senseval2.d001.s027.t002","gold_sense_keys":["concerned%3:00:00::"],"candidates":[{"index":1,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":2,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concerned%3:00:00::","call_ids":["senseval2.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concerned%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021155,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.812833083968144}
{"item_id":"senseval2.d001.s027.t003","gold_sense_keys":["advance%1:11:01::"],"candidates":[{"index":1,"sense_key":"advance%1:11:00::","synset_id":"progress.n.03"},{"index":2,"sense_key":"advance%1:11:01::","synset_id":"improvement.n.01"},{"index":3,"sense_key":"advance%1:10:00::","synset_id":"overture.n.03"},{"index":4,"sense_key":"advance%1:04:00::","synset_id":"progress.n.02"},{"index":5,"sense_key":"advance%1:21:00::","synset_id":"advance.n.05"},{"index":6,"sense_key":"advance%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"advance%1:11:01::","call_ids":["senseval2.d001.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"advance%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000317,"input_uncached_usd":0.000029,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.962493375001941}
{"item_id":"senseval2.d001.s027.t004","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["senseval2.d001.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016094999999999998,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.588980874978006}
{"item_id":"senseval2.d001.s027.t005","gold_sense_keys":["basic%3:00:00::","basic%5:00:02:first:00"],"candidates":[{"index":1,"sense_key":"basic%3:00:00::","synset_id":"basic.a.01"},{"index":2,"sense_key":"basic%5:00:00:standard:01","synset_id":"basic.s.01"},{"index":3,"sense_key":"basic%5:00:02:first:00","synset_id":"basic.s.02"},{"index":4,"sense_key":"basic%5:00:00:alkaline:00","synset_id":"basic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"basic%3:00:00::","call_ids":["senseval2.d001.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"basic%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002118,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.482511167007033}
{"item_id":"senseval2.d001.s027.t007","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"begin%2:30:01::","call_ids":["senseval2.d001.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"begin%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005359500000000001,"input_uncached_usd":0.00004315,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.789685083029326}
{"item_id":"senseval2.d001.s027.t008","gold_sense_keys":["show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"show%2:39:01::","call_ids":["senseval2.d001.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"show%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009743,"input_uncached_usd":0.0000463,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.28609150002012}
{"item_id":"senseval2.d001.s027.t009","gold_sense_keys":["result%1:11:00::","result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["senseval2.d001.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021385,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1657460419810377}
{"item_id":"senseval2.d001.s027.t010","gold_sense_keys":["there%4:02:02::"],"candidates":[{"index":1,"sense_key":"there%4:02:00::","synset_id":"there.r.01"},{"index":2,"sense_key":"there%4:02:02::","synset_id":"there.r.02"},{"index":3,"sense_key":"there%4:02:01::","synset_id":"there.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"there%4:02:00::","call_ids":["senseval2.d001.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"there%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005424999999999999,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.80099150002934}
{"item_id":"senseval2.d001.s029.t000","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval2.d001.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2292,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00037699999999999995,"input_uncached_usd":0.0001146,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.490833791962359}
{"item_id":"senseval2.d001.s029.t002","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["senseval2.d001.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049045,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.327964125026483}
{"item_id":"senseval2.d001.s030.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038855,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.477993417007383}
{"item_id":"senseval2.d001.s030.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035154999999999995,"input_uncached_usd":0.00003795,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.818615207972471}
{"item_id":"senseval2.d001.s030.t002","gold_sense_keys":["spring%1:28:00::"],"candidates":[{"index":1,"sense_key":"spring%1:28:00::","synset_id":"spring.n.01"},{"index":2,"sense_key":"spring%1:06:00::","synset_id":"spring.n.02"},{"index":3,"sense_key":"spring%1:17:00::","synset_id":"spring.n.03"},{"index":4,"sense_key":"spring%1:15:00::","synset_id":"spring.n.04"},{"index":5,"sense_key":"spring%1:07:00::","synset_id":"give.n.01"},{"index":6,"sense_key":"spring%1:04:00::","synset_id":"leap.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spring%1:28:00::","call_ids":["senseval2.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spring%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036784999999999997,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.179491915972903}
{"item_id":"senseval2.d001.s030.t004","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013885,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.298075958970003}
{"item_id":"senseval2.d001.s030.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006521000000000001,"input_uncached_usd":0.0000569,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.972135542018805}
{"item_id":"senseval2.d001.s030.t007","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s030.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1524,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0002362,"input_uncached_usd":0.0000762,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1403295419877395}
{"item_id":"senseval2.d001.s030.t008","gold_sense_keys":["impair%2:30:00::"],"candidates":[{"index":1,"sense_key":"impair%2:30:00::","synset_id":"impair.v.01"},{"index":2,"sense_key":"impair%2:30:01::","synset_id":"mar.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impair%2:30:00::","call_ids":["senseval2.d001.s030.t008__v1__a1","senseval2.d001.s030.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impair%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":881,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00074805,"input_uncached_usd":0.00004405,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.313767499988899}
{"item_id":"senseval2.d001.s030.t009","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"turn%2:30:03::","call_ids":["senseval2.d001.s030.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"turn%2:30:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1471,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00061755,"input_uncached_usd":0.00007355,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.648379417019896}
{"item_id":"senseval2.d001.s030.t010","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s030.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024055,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7232960839755833}
{"item_id":"senseval2.d001.s030.t011","gold_sense_keys":["colon%1:08:00::"],"candidates":[{"index":1,"sense_key":"colon%1:08:00::","synset_id":"colon.n.01"},{"index":2,"sense_key":"colon%1:23:01::","synset_id":"colon.n.02"},{"index":3,"sense_key":"colon%1:23:00::","synset_id":"colon.n.03"},{"index":4,"sense_key":"colon%1:15:00::","synset_id":"colon.n.04"},{"index":5,"sense_key":"colon%1:10:00::","synset_id":"colon.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"colon%1:08:00::","call_ids":["senseval2.d001.s030.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"colon%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021279999999999997,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8080879999906756}
{"item_id":"senseval2.d001.s030.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s030.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013974999999999998,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4126097920234315}
{"item_id":"senseval2.d001.s030.t013","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s030.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023304999999999997,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7898715000483207}
{"item_id":"senseval2.d001.s031.t001","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["senseval2.d001.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005511,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.716716040973552}
{"item_id":"senseval2.d001.s031.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015959999999999998,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.569495707983151}
{"item_id":"senseval2.d001.s031.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s031.t004__v1__a1","senseval2.d001.s031.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038655,"input_uncached_usd":0.00004094999999999999,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.397071249957662}
{"item_id":"senseval2.d001.s031.t005","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evidence%1:10:01::","call_ids":["senseval2.d001.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evidence%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005666,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.879104499996174}
{"item_id":"senseval2.d001.s031.t006","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003894,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.209679624997079}
{"item_id":"senseval2.d001.s031.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s031.t007__v1__a1","senseval2.d001.s031.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00062015,"input_uncached_usd":0.00004415,"input_cached_usd":0.0,"output_usd":0.0005759999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.369380833057221}
{"item_id":"senseval2.d001.s031.t009","gold_sense_keys":["present%3:00:02::"],"candidates":[{"index":1,"sense_key":"present%3:00:01::","synset_id":"present.a.01"},{"index":2,"sense_key":"present%3:00:02::","synset_id":"present.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"present%3:00:02::","call_ids":["senseval2.d001.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"present%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023189999999999997,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.609188999980688}
{"item_id":"senseval2.d001.s031.t010","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"},{"index":2,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028205,"input_uncached_usd":0.00001965,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.535360292007681}
{"item_id":"senseval2.d001.s032.t004","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d001.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020839999999999997,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.576991082984023}
{"item_id":"senseval2.d001.s032.t005","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"common%5:00:00:familiar:02","call_ids":["senseval2.d001.s032.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"common%5:00:00:familiar:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":1106,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048054999999999994,"input_uncached_usd":0.00003815,"input_cached_usd":0.0,"output_usd":0.00044239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.9194471659720875}
{"item_id":"senseval2.d001.s032.t007","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%1:09:01::","call_ids":["senseval2.d001.s032.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00021275,"input_uncached_usd":0.000052749999999999995,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9621721659787}
{"item_id":"senseval2.d001.s032.t010","gold_sense_keys":["kill%2:35:02::"],"candidates":[{"index":1,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":2,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":3,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":4,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":5,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":6,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":7,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":8,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":9,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":10,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":11,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"},{"index":12,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":13,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":14,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":15,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"kill%2:35:01::","call_ids":["senseval2.d001.s032.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"kill%2:35:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00081935,"input_uncached_usd":0.000044949999999999995,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.231212624988984}
{"item_id":"senseval2.d001.s032.t012","gold_sense_keys":["american%1:18:00::"],"candidates":[{"index":1,"sense_key":"american%1:18:00::","synset_id":"american.n.01"},{"index":2,"sense_key":"american%1:10:00::","synset_id":"american_english.n.01"},{"index":3,"sense_key":"american%1:18:01::","synset_id":"american.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%1:18:00::","call_ids":["senseval2.d001.s032.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004111,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.396210375009105}
{"item_id":"senseval2.d001.s032.t013","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d001.s032.t013__v1__a1","senseval2.d001.s032.t013__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1081,"cached_input_tokens":0,"output_tokens":2528,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.00106525,"input_uncached_usd":0.00005405,"input_cached_usd":0.0,"output_usd":0.0010111999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.791902041935828}
{"item_id":"senseval2.d001.s033.t000","gold_sense_keys":["right%4:02:04::","right%4:02:08::"],"candidates":[{"index":1,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":2,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":3,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":4,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":5,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":6,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":7,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":8,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":9,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":10,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"right%4:02:08::","call_ids":["senseval2.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"right%4:02:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032145000000000003,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.506663625012152}
{"item_id":"senseval2.d001.s033.t001","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029255,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.617755500017665}
{"item_id":"senseval2.d001.s033.t002","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034445,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.397384250012692}
{"item_id":"senseval2.d001.s033.t004","gold_sense_keys":["laboratory%1:06:00::"],"candidates":[{"index":1,"sense_key":"laboratory%1:06:00::","synset_id":"lab.n.01"},{"index":2,"sense_key":"laboratory%1:15:00::","synset_id":"testing_ground.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laboratory%1:06:00::","call_ids":["senseval2.d001.s033.t004__v1__a1","senseval2.d001.s033.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laboratory%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041385,"input_uncached_usd":0.00004264999999999999,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.925301834009588}
{"item_id":"senseval2.d001.s033.t005","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["senseval2.d001.s033.t005__v1__a1","senseval2.d001.s033.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005428499999999999,"input_uncached_usd":0.000043650000000000004,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.997804083977826}
{"item_id":"senseval2.d001.s033.t008","gold_sense_keys":["race%2:41:03::"],"candidates":[{"index":1,"sense_key":"race%2:38:00::","synset_id":"rush.v.01"},{"index":2,"sense_key":"race%2:33:00::","synset_id":"race.v.02"},{"index":3,"sense_key":"race%2:41:03::","synset_id":"race.v.03"},{"index":4,"sense_key":"race%2:38:10::","synset_id":"race.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"race%2:41:03::","call_ids":["senseval2.d001.s033.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"race%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013289999999999998,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3912135410355404}
{"item_id":"senseval2.d001.s033.t009","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s033.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020899999999999998,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.102913165988866}
{"item_id":"senseval2.d001.s034.t000","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037144999999999995,"input_uncached_usd":0.00003225,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.755793916992843}
{"item_id":"senseval2.d001.s035.t001","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038034999999999995,"input_uncached_usd":0.00004115,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.034823833964765}
{"item_id":"senseval2.d001.s035.t003","gold_sense_keys":["alone%4:02:01::"],"candidates":[{"index":1,"sense_key":"alone%4:02:00::","synset_id":"entirely.r.02"},{"index":2,"sense_key":"alone%4:02:01::","synset_id":"alone.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"alone%4:02:01::","call_ids":["senseval2.d001.s035.t003__v1__a1","senseval2.d001.s035.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"alone%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":5601,"reasoning_output_tokens":5568},"cost":{"currency":"USD","total_usd":0.00228695,"input_uncached_usd":0.00004654999999999999,"input_cached_usd":0.0,"output_usd":0.0022404,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":37.7020291670342}
{"item_id":"senseval2.d001.s035.t004","gold_sense_keys":["combination%1:14:00::"],"candidates":[{"index":1,"sense_key":"combination%1:14:00::","synset_id":"combination.n.01"},{"index":2,"sense_key":"combination%1:14:04::","synset_id":"combination.n.02"},{"index":3,"sense_key":"combination%1:14:02::","synset_id":"combination.n.03"},{"index":4,"sense_key":"combination%1:14:01::","synset_id":"combination.n.04"},{"index":5,"sense_key":"combination%1:14:03::","synset_id":"combination.n.05"},{"index":6,"sense_key":"combination%1:04:01::","synset_id":"combination.n.06"},{"index":7,"sense_key":"combination%1:04:00::","synset_id":"combination.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"combination%1:14:00::","call_ids":["senseval2.d001.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"combination%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006244,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.127143417019397}
{"item_id":"senseval2.d001.s035.t005","gold_sense_keys":["appear%2:39:00::","appear%2:39:01::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appear%2:39:01::","call_ids":["senseval2.d001.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appear%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014309999999999998,"input_uncached_usd":0.0000343,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5928670419962145}
{"item_id":"senseval2.d001.s035.t006","gold_sense_keys":["crucial%5:00:00:important:00","crucial%5:00:00:material:02"],"candidates":[{"index":1,"sense_key":"crucial%3:00:00::","synset_id":"crucial.a.01"},{"index":2,"sense_key":"crucial%5:00:00:material:02","synset_id":"crucial.s.01"},{"index":3,"sense_key":"crucial%5:00:00:important:00","synset_id":"all-important.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"crucial%5:00:00:important:00","call_ids":["senseval2.d001.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"crucial%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002606,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.360378417011816}
{"item_id":"senseval2.d001.s035.t007","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066205,"input_uncached_usd":0.00004085,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.467273667047266}
{"item_id":"senseval2.d001.s035.t009","gold_sense_keys":["scourge%1:26:00::"],"candidates":[{"index":1,"sense_key":"scourge%1:06:00::","synset_id":"scourge.n.01"},{"index":2,"sense_key":"scourge%1:26:00::","synset_id":"bane.n.01"},{"index":3,"sense_key":"scourge%1:18:00::","synset_id":"terror.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scourge%1:26:00::","call_ids":["senseval2.d001.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scourge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043904999999999997,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.318257958977483}
{"item_id":"senseval2.d001.s035.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018705,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.253239250043407}
{"item_id":"senseval2.d001.s035.t011","gold_sense_keys":["brain%1:08:00::"],"candidates":[{"index":1,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":2,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":5,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brain%1:08:00::","call_ids":["senseval2.d001.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brain%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023979999999999997,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.898947749985382}
{"item_id":"senseval2.d001.s035.t012","gold_sense_keys":["skin%1:08:00::"],"candidates":[{"index":1,"sense_key":"skin%1:08:00::","synset_id":"skin.n.01"},{"index":2,"sense_key":"skin%1:06:01::","synset_id":"skin.n.02"},{"index":3,"sense_key":"skin%1:05:01::","synset_id":"hide.n.02"},{"index":4,"sense_key":"skin%1:26:00::","synset_id":"skin.n.04"},{"index":5,"sense_key":"skin%1:13:00::","synset_id":"peel.n.02"},{"index":6,"sense_key":"skin%1:06:02::","synset_id":"skin.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"skin%1:08:00::","call_ids":["senseval2.d001.s035.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"skin%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002384,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2452114170300774}
{"item_id":"senseval2.d001.s035.t015","gold_sense_keys":["cervix%1:08:02::"],"candidates":[{"index":1,"sense_key":"cervix%1:08:01::","synset_id":"neck.n.01"},{"index":2,"sense_key":"cervix%1:08:02::","synset_id":"cervix.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cervix%1:08:02::","call_ids":["senseval2.d001.s035.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cervix%1:08:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023299999999999997,"input_uncached_usd":0.000021799999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8244279170176014}
{"item_id":"senseval2.d001.s036.t000","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036329999999999994,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.237019749998581}
{"item_id":"senseval2.d001.s036.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d001.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001613,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.871611375012435}
{"item_id":"senseval2.d001.s036.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010624999999999998,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.8970925839967094}
{"item_id":"senseval2.d001.s036.t004","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021605,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8019274590187706}
{"item_id":"senseval2.d001.s036.t006","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s036.t006__v1__a1","senseval2.d001.s036.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041785,"input_uncached_usd":0.000046649999999999996,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5794437500298955}
{"item_id":"senseval2.d001.s036.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011119999999999999,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5779646250302903}
{"item_id":"senseval2.d001.s036.t009","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"explain%2:32:02::","call_ids":["senseval2.d001.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"explain%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003127,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.721128833014518}
{"item_id":"senseval2.d001.s036.t010","gold_sense_keys":["finally%4:02:02::"],"candidates":[{"index":1,"sense_key":"finally%4:02:02::","synset_id":"finally.r.01"},{"index":2,"sense_key":"finally%4:02:00::","synset_id":"ultimately.r.01"},{"index":3,"sense_key":"finally%4:02:03::","synset_id":"last.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"finally%4:02:00::","call_ids":["senseval2.d001.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"finally%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007478,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.386143292009365}
{"item_id":"senseval2.d001.s036.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016245,"input_uncached_usd":0.000028049999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.251182375010103}
{"item_id":"senseval2.d001.s036.t012","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval2.d001.s036.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004432,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.01779275003355}
{"item_id":"senseval2.d001.s036.t014","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s036.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037329999999999997,"input_uncached_usd":0.000034099999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.059037958038971}
{"item_id":"senseval2.d001.s036.t015","gold_sense_keys":["family%1:14:01::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"family%1:14:01::","call_ids":["senseval2.d001.s036.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"family%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007596,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.746976541006006}
{"item_id":"senseval2.d001.s037.t000","gold_sense_keys":["story%1:10:04::"],"candidates":[{"index":1,"sense_key":"story%1:10:03::","synset_id":"narrative.n.01"},{"index":2,"sense_key":"story%1:10:00::","synset_id":"story.n.02"},{"index":3,"sense_key":"story%1:06:00::","synset_id":"floor.n.02"},{"index":4,"sense_key":"story%1:10:04::","synset_id":"history.n.02"},{"index":5,"sense_key":"story%1:10:01::","synset_id":"report.n.03"},{"index":6,"sense_key":"story%1:10:02::","synset_id":"fib.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"story%1:10:04::","call_ids":["senseval2.d001.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"story%1:10:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":682,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021969999999999997,"input_uncached_usd":0.000034099999999999995,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.413841042027343}
{"item_id":"senseval2.d001.s037.t002","gold_sense_keys":["go_back%2:42:00::"],"candidates":[{"index":1,"sense_key":"go_back%2:42:00::","synset_id":"go_back.v.01"},{"index":2,"sense_key":"go_back%2:32:00::","synset_id":"recur.v.02"},{"index":3,"sense_key":"go_back%2:30:00::","synset_id":"recover.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go_back%2:42:00::","call_ids":["senseval2.d001.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go_back%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013324999999999999,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7906015409971587}
{"item_id":"senseval2.d001.s037.t005","gold_sense_keys":["name%2:32:03::"],"candidates":[{"index":1,"sense_key":"name%2:32:03::","synset_id":"name.v.01"},{"index":2,"sense_key":"name%2:32:02::","synset_id":"name.v.02"},{"index":3,"sense_key":"name%2:41:01::","synset_id":"name.v.03"},{"index":4,"sense_key":"name%2:41:00::","synset_id":"appoint.v.01"},{"index":5,"sense_key":"name%2:32:07::","synset_id":"name.v.05"},{"index":6,"sense_key":"name%2:32:04::","synset_id":"mention.v.01"},{"index":7,"sense_key":"name%2:31:01::","synset_id":"identify.v.05"},{"index":8,"sense_key":"name%2:32:09::","synset_id":"list.v.01"},{"index":9,"sense_key":"name%2:31:00::","synset_id":"diagnose.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%2:32:03::","call_ids":["senseval2.d001.s037.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%2:32:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029989999999999997,"input_uncached_usd":0.0000375,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.689203542016912}
{"item_id":"senseval2.d001.s037.t006","gold_sense_keys":["propose%2:31:00::"],"candidates":[{"index":1,"sense_key":"propose%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"propose%2:31:00::","synset_id":"project.v.08"},{"index":3,"sense_key":"propose%2:31:01::","synset_id":"aim.v.02"},{"index":4,"sense_key":"propose%2:41:00::","synset_id":"nominate.v.02"},{"index":5,"sense_key":"propose%2:32:01::","synset_id":"propose.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"propose%2:31:00::","call_ids":["senseval2.d001.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"propose%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.000549,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.249600917042699}
{"item_id":"senseval2.d001.s037.t008","gold_sense_keys":["stem%2:42:00::"],"candidates":[{"index":1,"sense_key":"stem%2:42:00::","synset_id":"stem.v.01"},{"index":2,"sense_key":"stem%2:42:03::","synset_id":"stem.v.02"},{"index":3,"sense_key":"stem%2:30:00::","synset_id":"stem.v.03"},{"index":4,"sense_key":"stem%2:30:01::","synset_id":"stem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stem%2:42:00::","call_ids":["senseval2.d001.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056955,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.046612125006504}
{"item_id":"senseval2.d001.s037.t009","gold_sense_keys":["separate%3:00:00::"],"candidates":[{"index":1,"sense_key":"separate%3:00:00::","synset_id":"separate.a.01"},{"index":2,"sense_key":"separate%5:00:00:detached:00","synset_id":"freestanding.s.01"},{"index":3,"sense_key":"separate%5:00:02:segregated:00","synset_id":"separate.s.02"},{"index":4,"sense_key":"separate%5:00:02:unconnected:00","synset_id":"disjoined.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"separate%3:00:00::","call_ids":["senseval2.d001.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"separate%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034044999999999995,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.967689374985639}
{"item_id":"senseval2.d001.s037.t010","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s037.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000315,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.21202566701686}
{"item_id":"senseval2.d001.s037.t011","gold_sense_keys":["defect%1:26:01::"],"candidates":[{"index":1,"sense_key":"defect%1:26:01::","synset_id":"defect.n.01"},{"index":2,"sense_key":"defect%1:07:01::","synset_id":"defect.n.02"},{"index":3,"sense_key":"defect%1:26:00::","synset_id":"defect.n.03"},{"index":4,"sense_key":"defect%1:07:00::","synset_id":"blemish.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"defect%1:07:01::","call_ids":["senseval2.d001.s037.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"defect%1:07:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00069925,"input_uncached_usd":0.000027249999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.034613416995853}
{"item_id":"senseval2.d001.s038.t000","gold_sense_keys":["theorize%2:31:00::"],"candidates":[{"index":1,"sense_key":"theorize%2:31:00::","synset_id":"speculate.v.01"},{"index":2,"sense_key":"theorize%2:31:02::","synset_id":"theorize.v.02"},{"index":3,"sense_key":"theorize%2:31:01::","synset_id":"theorize.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"theorize%2:31:02::","call_ids":["senseval2.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"theorize%2:31:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038885,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.567647624993697}
{"item_id":"senseval2.d001.s038.t001","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018734999999999997,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5138637920026667}
{"item_id":"senseval2.d001.s038.t002","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003411,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.01588541601086}
{"item_id":"senseval2.d001.s038.t004","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s038.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000802,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.6260620420216583}
{"item_id":"senseval2.d001.s038.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028429999999999997,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.751544625032693}
{"item_id":"senseval2.d001.s038.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036419999999999996,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.721133875020314}
{"item_id":"senseval2.d001.s038.t008","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s038.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036054999999999995,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.556997417006642}
{"item_id":"senseval2.d001.s038.t009","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s038.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023554999999999998,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.478990832983982}
{"item_id":"senseval2.d001.s038.t010","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s038.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026179999999999997,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.381175332993735}
{"item_id":"senseval2.d001.s039.t002","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1283,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00058255,"input_uncached_usd":0.00006415,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.480271915963385}
{"item_id":"senseval2.d001.s039.t003","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001814,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9046630410011858}
{"item_id":"senseval2.d001.s039.t004","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013269999999999997,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.441188708995469}
{"item_id":"senseval2.d001.s039.t005","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041535,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.156936625018716}
{"item_id":"senseval2.d001.s039.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s039.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000207,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0010304169845767}
{"item_id":"senseval2.d001.s040.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval2.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003301,"input_uncached_usd":0.0000421,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.279818999988493}
{"item_id":"senseval2.d001.s040.t001","gold_sense_keys":["prove%2:31:00::"],"candidates":[{"index":1,"sense_key":"prove%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"prove%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"prove%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"prove%2:31:02::","synset_id":"prove.v.04"},{"index":5,"sense_key":"prove%2:41:03::","synset_id":"test.v.01"},{"index":6,"sense_key":"prove%2:38:01::","synset_id":"rise.v.15"},{"index":7,"sense_key":"prove%2:38:00::","synset_id":"raise.v.15"},{"index":8,"sense_key":"prove%2:36:00::","synset_id":"prove.v.08"},{"index":9,"sense_key":"prove%2:31:01::","synset_id":"prove.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prove%2:31:00::","call_ids":["senseval2.d001.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prove%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040219999999999996,"input_uncached_usd":0.0000374,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.518007541017141}
{"item_id":"senseval2.d001.s040.t002","gold_sense_keys":["theory%1:09:01::"],"candidates":[{"index":1,"sense_key":"theory%1:09:00::","synset_id":"theory.n.01"},{"index":2,"sense_key":"theory%1:09:01::","synset_id":"hypothesis.n.02"},{"index":3,"sense_key":"theory%1:09:02::","synset_id":"theory.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"theory%1:09:01::","call_ids":["senseval2.d001.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"theory%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007002499999999999,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.495020125003066}
{"item_id":"senseval2.d001.s041.t000","gold_sense_keys":["back%4:02:01::"],"candidates":[{"index":1,"sense_key":"back%4:02:04::","synset_id":"back.r.01"},{"index":2,"sense_key":"back%4:02:00::","synset_id":"back.r.02"},{"index":3,"sense_key":"back%4:02:05::","synset_id":"back.r.03"},{"index":4,"sense_key":"back%4:02:01::","synset_id":"back.r.04"},{"index":5,"sense_key":"back%4:02:06::","synset_id":"back.r.05"},{"index":6,"sense_key":"back%4:02:03::","synset_id":"back.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"back%4:02:01::","call_ids":["senseval2.d001.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"back%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001881,"input_uncached_usd":0.0000281,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.838281874952372}
{"item_id":"senseval2.d001.s041.t001","gold_sense_keys":["then%4:02:01::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000808,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.8450103340437636}
{"item_id":"senseval2.d001.s041.t003","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval2.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030274999999999996,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.084902082977351}
{"item_id":"senseval2.d001.s041.t008","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval2.d001.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1480,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00015719999999999997,"input_uncached_usd":0.000074,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9620136669836938}
{"item_id":"senseval2.d001.s041.t009","gold_sense_keys":["pair%1:14:01::"],"candidates":[{"index":1,"sense_key":"pair%1:14:01::","synset_id":"pair.n.01"},{"index":2,"sense_key":"pair%1:23:00::","synset_id":"couple.n.04"},{"index":3,"sense_key":"pair%1:14:00::","synset_id":"pair.n.03"},{"index":4,"sense_key":"pair%1:14:02::","synset_id":"pair.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pair%1:14:01::","call_ids":["senseval2.d001.s041.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pair%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006180000000000001,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.30211175000295}
{"item_id":"senseval2.d001.s041.t011","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s041.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024165,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.566347665968351}
{"item_id":"senseval2.d001.s041.t012","gold_sense_keys":["contain%2:42:13::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"contain%2:42:13::","call_ids":["senseval2.d001.s041.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"contain%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024309999999999997,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.149322709010448}
{"item_id":"senseval2.d001.s042.t001","gold_sense_keys":["gross%5:00:00:visible:00"],"candidates":[{"index":1,"sense_key":"gross%3:00:00::","synset_id":"gross.a.01"},{"index":2,"sense_key":"gross%5:00:00:general:00","synset_id":"gross.s.01"},{"index":3,"sense_key":"gross%5:00:00:fat:01","synset_id":"gross.s.02"},{"index":4,"sense_key":"gross%5:00:00:visible:00","synset_id":"megascopic.s.01"},{"index":5,"sense_key":"gross%5:00:00:unmitigated:00","synset_id":"arrant.s.01"},{"index":6,"sense_key":"gross%5:00:00:indecent:00","synset_id":"crude.s.02"},{"index":7,"sense_key":"gross%5:00:00:conspicuous:00","synset_id":"crying.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"gross%5:00:00:visible:00","call_ids":["senseval2.d001.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"gross%5:00:00:visible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042325,"input_uncached_usd":0.000032449999999999996,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.409014708013274}
{"item_id":"senseval2.d001.s042.t003","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031340000000000003,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.417866999981925}
{"item_id":"senseval2.d001.s042.t004","gold_sense_keys":["visible%3:00:00::"],"candidates":[{"index":1,"sense_key":"visible%3:00:00::","synset_id":"visible.a.01"},{"index":2,"sense_key":"visible%5:00:00:overt:00","synset_id":"visible.s.01"},{"index":3,"sense_key":"visible%5:00:00:available:00","synset_id":"visible.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"visible%3:00:00::","call_ids":["senseval2.d001.s042.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"visible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023385,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.946553042042069}
{"item_id":"senseval2.d001.s043.t000","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1100,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004454,"input_uncached_usd":0.000054999999999999995,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.1618807499762625}
{"item_id":"senseval2.d001.s043.t001","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015849999999999998,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6512203330057673}
{"item_id":"senseval2.d001.s043.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021084999999999998,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.386369374988135}
{"item_id":"senseval2.d001.s043.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s043.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018544999999999998,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.032174499996472}
{"item_id":"senseval2.d001.s043.t004","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s043.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000781,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6711762500344776}
{"item_id":"senseval2.d001.s043.t005","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s043.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005894000000000001,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.26471383398166}
{"item_id":"senseval2.d001.s043.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s043.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023409999999999997,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.89913583395537}
{"item_id":"senseval2.d001.s043.t009","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d001.s043.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030725,"input_uncached_usd":0.00001925,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.841747500002384}
{"item_id":"senseval2.d001.s044.t001","gold_sense_keys":["actually%4:02:01::"],"candidates":[{"index":1,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"},{"index":2,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":3,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":4,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"actually%4:02:01::","call_ids":["senseval2.d001.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"actually%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004145,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.631769000028726}
{"item_id":"senseval2.d001.s044.t002","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval2.d001.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1445,"cached_input_tokens":0,"output_tokens":914,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00043785,"input_uncached_usd":0.00007225,"input_cached_usd":0.0,"output_usd":0.0003656,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.334092625009362}
{"item_id":"senseval2.d001.s044.t003","gold_sense_keys":["bit%1:17:00::","bit%1:23:01::"],"candidates":[{"index":1,"sense_key":"bit%1:23:01::","synset_id":"spot.n.10"},{"index":2,"sense_key":"bit%1:17:00::","synset_id":"bit.n.02"},{"index":3,"sense_key":"bit%1:28:00::","synset_id":"moment.n.02"},{"index":4,"sense_key":"bit%1:11:00::","synset_id":"piece.n.05"},{"index":5,"sense_key":"bit%1:06:01::","synset_id":"bit.n.05"},{"index":6,"sense_key":"bit%1:23:00::","synset_id":"bit.n.06"},{"index":7,"sense_key":"bit%1:13:00::","synset_id":"morsel.n.02"},{"index":8,"sense_key":"bit%1:10:01::","synset_id":"snatch.n.01"},{"index":9,"sense_key":"bit%1:10:00::","synset_id":"act.n.04"},{"index":10,"sense_key":"bit%1:06:02::","synset_id":"bit.n.10"},{"index":11,"sense_key":"bit%1:06:00::","synset_id":"bit.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bit%1:17:00::","call_ids":["senseval2.d001.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bit%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035305,"input_uncached_usd":0.00003945,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.796760458033532}
{"item_id":"senseval2.d001.s045.t000","gold_sense_keys":["assume%2:31:00::"],"candidates":[{"index":1,"sense_key":"assume%2:31:00::","synset_id":"assume.v.01"},{"index":2,"sense_key":"assume%2:41:00::","synset_id":"assume.v.02"},{"index":3,"sense_key":"assume%2:30:00::","synset_id":"assume.v.03"},{"index":4,"sense_key":"assume%2:40:01::","synset_id":"bear.v.06"},{"index":5,"sense_key":"assume%2:38:00::","synset_id":"assume.v.05"},{"index":6,"sense_key":"assume%2:40:00::","synset_id":"assume.v.06"},{"index":7,"sense_key":"assume%2:36:00::","synset_id":"simulate.v.03"},{"index":8,"sense_key":"assume%2:35:00::","synset_id":"assume.v.08"},{"index":9,"sense_key":"assume%2:29:00::","synset_id":"wear.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assume%2:31:00::","call_ids":["senseval2.d001.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assume%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002497,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5580414590076543}
{"item_id":"senseval2.d001.s045.t002","gold_sense_keys":["piece%1:06:00::","piece%1:17:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"piece%1:06:00::","call_ids":["senseval2.d001.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"piece%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.000254,"input_uncached_usd":0.0000428,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8412905420409516}
{"item_id":"senseval2.d001.s045.t003","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"contain%2:42:13::","call_ids":["senseval2.d001.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"contain%2:42:13::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031790000000000003,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.28250345896231}
{"item_id":"senseval2.d001.s045.t006","gold_sense_keys":["loss%1:07:00::","loss%1:21:01::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":2257,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009339999999999999,"input_uncached_usd":0.0000312,"input_cached_usd":0.0,"output_usd":0.0009027999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.780546291964129}
{"item_id":"senseval2.d001.s045.t007","gold_sense_keys":["critical%5:00:00:indispensable:00"],"candidates":[{"index":1,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":2,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":3,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":4,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":5,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":6,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":7,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"critical%5:00:00:indispensable:00","call_ids":["senseval2.d001.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"critical%5:00:00:indispensable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005993000000000001,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.907963541976642}
{"item_id":"senseval2.d001.s045.t008","gold_sense_keys":["role%1:07:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"role%1:07:00::","call_ids":["senseval2.d001.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"role%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015879999999999998,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4733934159739874}
{"item_id":"senseval2.d001.s045.t009","gold_sense_keys":["set_off%2:36:00::"],"candidates":[{"index":1,"sense_key":"set_off%2:36:00::","synset_id":"trip.v.04"},{"index":2,"sense_key":"set_off%2:38:00::","synset_id":"depart.v.03"},{"index":3,"sense_key":"set_off%2:30:02::","synset_id":"bring_out.v.04"},{"index":4,"sense_key":"set_off%2:30:00::","synset_id":"explode.v.01"},{"index":5,"sense_key":"set_off%2:42:00::","synset_id":"cancel.v.02"},{"index":6,"sense_key":"set_off%2:30:01::","synset_id":"set_off.v.06"},{"index":7,"sense_key":"set_off%2:32:00::","synset_id":"incite.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set_off%2:36:00::","call_ids":["senseval2.d001.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set_off%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034409999999999996,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.370929291006178}
{"item_id":"senseval2.d001.s045.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001848,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2001989579875953}
{"item_id":"senseval2.d001.s046.t000","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d001.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.000252,"input_uncached_usd":0.000040799999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8484168329741806}
{"item_id":"senseval2.d001.s047.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["senseval2.d001.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030925,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.458634874958079}
{"item_id":"senseval2.d001.s047.t001","gold_sense_keys":["scientific%3:01:00::"],"candidates":[{"index":1,"sense_key":"scientific%3:01:00::","synset_id":"scientific.a.01"},{"index":2,"sense_key":"scientific%3:00:00::","synset_id":"scientific.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scientific%3:01:00::","call_ids":["senseval2.d001.s047.t001__v1__a1","senseval2.d001.s047.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scientific%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":1569,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066625,"input_uncached_usd":0.000038650000000000004,"input_cached_usd":0.0,"output_usd":0.0006276,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.770822667051107}
{"item_id":"senseval2.d001.s047.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005872500000000001,"input_uncached_usd":0.00001765,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.161194416985381}
{"item_id":"senseval2.d001.s047.t003","gold_sense_keys":["lead%2:41:00::"],"candidates":[{"index":1,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":2,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":3,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":4,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":5,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":6,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":7,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":8,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":9,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":10,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":11,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":12,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":13,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":14,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"lead%2:41:00::","call_ids":["senseval2.d001.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"lead%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":878,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008439000000000001,"input_uncached_usd":0.000043899999999999996,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.57891662500333}
{"item_id":"senseval2.d001.s047.t006","gold_sense_keys":["then%4:02:01::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d001.s047.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004117,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.957440791011322}
{"item_id":"senseval2.d001.s047.t007","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:40:03::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s047.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1089,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.00100805,"input_uncached_usd":0.000054449999999999995,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.439939500007313}
{"item_id":"senseval2.d001.s047.t008","gold_sense_keys":["answer%1:10:02::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"answer%1:10:02::","call_ids":["senseval2.d001.s047.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"answer%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009311,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.865292291971855}
{"item_id":"senseval2.d001.s048.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053715,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.470664458000101}
{"item_id":"senseval2.d001.s048.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029245,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.083374208013993}
{"item_id":"senseval2.d001.s048.t002","gold_sense_keys":["battery%1:14:01::"],"candidates":[{"index":1,"sense_key":"battery%1:14:02::","synset_id":"battery.n.01"},{"index":2,"sense_key":"battery%1:06:00::","synset_id":"battery.n.02"},{"index":3,"sense_key":"battery%1:14:01::","synset_id":"battery.n.03"},{"index":4,"sense_key":"battery%1:14:03::","synset_id":"battery.n.04"},{"index":5,"sense_key":"battery%1:06:03::","synset_id":"battery.n.05"},{"index":6,"sense_key":"battery%1:04:01::","synset_id":"barrage.n.02"},{"index":7,"sense_key":"battery%1:04:00::","synset_id":"battery.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"battery%1:14:01::","call_ids":["senseval2.d001.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"battery%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011235,"input_uncached_usd":0.000029149999999999997,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2129110420355573}
{"item_id":"senseval2.d001.s048.t004","gold_sense_keys":["develop%2:36:01::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"develop%2:36:01::","call_ids":["senseval2.d001.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"develop%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1261,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00060705,"input_uncached_usd":0.00006305,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.214235582971014}
{"item_id":"senseval2.d001.s048.t007","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s048.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002873,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.498808374977671}
{"item_id":"senseval2.d001.s048.t008","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":2,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":3,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":4,"sense_key":"material%1:06:01::","synset_id":"material.n.04"},{"index":5,"sense_key":"material%1:18:00::","synset_id":"material.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"material%1:27:00::","call_ids":["senseval2.d001.s048.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041835,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.614840459020343}
{"item_id":"senseval2.d001.s048.t011","gold_sense_keys":["presence%1:26:00::"],"candidates":[{"index":1,"sense_key":"presence%1:26:00::","synset_id":"presence.n.01"},{"index":2,"sense_key":"presence%1:15:00::","synset_id":"presence.n.02"},{"index":3,"sense_key":"presence%1:18:00::","synset_id":"presence.n.03"},{"index":4,"sense_key":"presence%1:09:00::","synset_id":"presence.n.04"},{"index":5,"sense_key":"presence%1:07:00::","synset_id":"bearing.n.03"},{"index":6,"sense_key":"presence%1:04:00::","synset_id":"presence.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"presence%1:18:00::","call_ids":["senseval2.d001.s048.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"presence%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036474999999999995,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.820793999999296}
{"item_id":"senseval2.d001.s048.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s048.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024074999999999997,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9953827079734765}
{"item_id":"senseval2.d001.s049.t000","gold_sense_keys":["analyze%2:31:00::","analyze%2:31:01::"],"candidates":[{"index":1,"sense_key":"analyze%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"analyze%2:31:01::","synset_id":"analyze.v.02"},{"index":3,"sense_key":"analyze%2:31:04::","synset_id":"analyze.v.03"},{"index":4,"sense_key":"analyze%2:31:03::","synset_id":"analyze.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"analyze%2:31:00::","call_ids":["senseval2.d001.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"analyze%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036424999999999994,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.30935941595817}
{"item_id":"senseval2.d001.s049.t001","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021425,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3237226249766536}
{"item_id":"senseval2.d001.s049.t002","gold_sense_keys":["extract%2:35:04::"],"candidates":[{"index":1,"sense_key":"extract%2:35:04::","synset_id":"extract.v.01"},{"index":2,"sense_key":"extract%2:40:00::","synset_id":"extract.v.02"},{"index":3,"sense_key":"extract%2:36:00::","synset_id":"educe.v.01"},{"index":4,"sense_key":"extract%2:30:00::","synset_id":"distill.v.03"},{"index":5,"sense_key":"extract%2:35:07::","synset_id":"extract.v.05"},{"index":6,"sense_key":"extract%2:35:05::","synset_id":"press_out.v.03"},{"index":7,"sense_key":"extract%2:31:00::","synset_id":"excerpt.v.01"},{"index":8,"sense_key":"extract%2:31:02::","synset_id":"extract.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extract%2:35:04::","call_ids":["senseval2.d001.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extract%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034445,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.366745625040494}
{"item_id":"senseval2.d001.s049.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval2.d001.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023564999999999998,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.673603583010845}
{"item_id":"senseval2.d001.s049.t005","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1084,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006242,"input_uncached_usd":0.000054199999999999996,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.328862334019504}
{"item_id":"senseval2.d001.s049.t006","gold_sense_keys":["defect%1:26:01::"],"candidates":[{"index":1,"sense_key":"defect%1:26:01::","synset_id":"defect.n.01"},{"index":2,"sense_key":"defect%1:07:01::","synset_id":"defect.n.02"},{"index":3,"sense_key":"defect%1:26:00::","synset_id":"defect.n.03"},{"index":4,"sense_key":"defect%1:07:00::","synset_id":"blemish.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"defect%1:07:01::","call_ids":["senseval2.d001.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"defect%1:07:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1809,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007474499999999999,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0007235999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.97395679098554}
{"item_id":"senseval2.d001.s049.t007","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003581999999999999,"input_uncached_usd":0.000018999999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.72776395798428}
{"item_id":"senseval2.d001.s049.t008","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001825,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1338105829781853}
{"item_id":"senseval2.d001.s049.t010","gold_sense_keys":["exact%3:00:00::"],"candidates":[{"index":1,"sense_key":"exact%3:00:00::","synset_id":"exact.a.01"},{"index":2,"sense_key":"exact%5:00:00:correct:00","synset_id":"accurate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exact%5:00:00:correct:00","call_ids":["senseval2.d001.s049.t010__v1__a1","senseval2.d001.s049.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exact%5:00:00:correct:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":3104,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.00128245,"input_uncached_usd":0.000040850000000000004,"input_cached_usd":0.0,"output_usd":0.0012416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":22.60422566701891}
{"item_id":"senseval2.d001.s049.t012","gold_sense_keys":["first%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d001.s049.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023874999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9998177920351736}
{"item_id":"senseval2.d001.s049.t013","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s049.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003357,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7004308750038035}
{"item_id":"senseval2.d001.s050.t000","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"},{"index":2,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":3,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finding%1:17:00::","call_ids":["senseval2.d001.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015534999999999998,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.539911750005558}
{"item_id":"senseval2.d001.s050.t001","gold_sense_keys":["medicine%1:04:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"medicine%1:04:00::","call_ids":["senseval2.d001.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"medicine%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001837,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.350307166983839}
{"item_id":"senseval2.d001.s051.t000","gold_sense_keys":["first%3:00:00::"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"first%5:00:02:opening:00","call_ids":["senseval2.d001.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"first%5:00:02:opening:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":2576,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010585999999999998,"input_uncached_usd":0.000028199999999999998,"input_cached_usd":0.0,"output_usd":0.0010304,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":19.69368658302119}
{"item_id":"senseval2.d001.s051.t001","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["senseval2.d001.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006563000000000001,"input_uncached_usd":0.000035499999999999996,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.734722541004885}
{"item_id":"senseval2.d001.s051.t002","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"show%2:31:00::","call_ids":["senseval2.d001.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"show%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038299999999999993,"input_uncached_usd":0.0000438,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.86392758297734}
{"item_id":"senseval2.d001.s051.t003","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021725,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0972005830262788}
{"item_id":"senseval2.d001.s051.t004","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033634999999999996,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.306145875016227}
{"item_id":"senseval2.d001.s051.t005","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s051.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023454999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.97149912500754}
{"item_id":"senseval2.d001.s051.t007","gold_sense_keys":["lead%2:41:12::","lead%2:42:12::"],"candidates":[{"index":1,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":2,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":3,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":4,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":5,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":6,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":7,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":8,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":9,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":10,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":11,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":12,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":13,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":14,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lead%2:42:12::","call_ids":["senseval2.d001.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lead%2:42:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017869999999999998,"input_uncached_usd":0.0000443,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.825967375014443}
{"item_id":"senseval2.d001.s051.t008","gold_sense_keys":["eruption%1:11:03::","eruption%1:26:02::"],"candidates":[{"index":1,"sense_key":"eruption%1:11:00::","synset_id":"volcanic_eruption.n.01"},{"index":2,"sense_key":"eruption%1:26:02::","synset_id":"eruption.n.02"},{"index":3,"sense_key":"eruption%1:26:01::","synset_id":"eruption.n.03"},{"index":4,"sense_key":"eruption%1:11:03::","synset_id":"outbreak.n.01"},{"index":5,"sense_key":"eruption%1:11:01::","synset_id":"bang.n.02"},{"index":6,"sense_key":"eruption%1:11:02::","synset_id":"eruption.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"eruption%1:11:03::","call_ids":["senseval2.d001.s051.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"eruption%1:11:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026284999999999997,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.574121958983596}
{"item_id":"senseval2.d001.s051.t009","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s051.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021084999999999998,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6480580419884063}
{"item_id":"senseval2.d001.s052.t001","gold_sense_keys":["satisfying%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"satisfying%5:00:00:wholesome:00","synset_id":"hearty.s.02"},{"index":2,"sense_key":"satisfying%5:00:00:satisfactory:00","synset_id":"comforting.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"satisfying%5:00:00:satisfactory:00","call_ids":["senseval2.d001.s052.t001__v1__a1","senseval2.d001.s052.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"satisfying%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066895,"input_uncached_usd":0.00004175,"input_cached_usd":0.0,"output_usd":0.0006272,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.41526654199697}
{"item_id":"senseval2.d001.s052.t002","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058375,"input_uncached_usd":0.00003975,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.848167790973093}
{"item_id":"senseval2.d001.s052.t003","gold_sense_keys":["now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021674999999999998,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.099864708026871}
{"item_id":"senseval2.d001.s053.t000","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":2,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":3,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":4,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":5,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":6,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":7,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":8,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":9,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":10,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":11,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":12,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"true%3:00:00::","call_ids":["senseval2.d001.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050795,"input_uncached_usd":0.00004075,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.306424041977152}
{"item_id":"senseval2.d001.s053.t002","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":2,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":3,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":4,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":5,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":6,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":7,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":8,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":9,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":10,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":11,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":12,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"true%3:00:00::","call_ids":["senseval2.d001.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030355,"input_uncached_usd":0.00004075,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.319896458007861}
{"item_id":"senseval2.d001.s053.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.000134,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0001092,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2309547499753535}
{"item_id":"senseval2.d001.s054.t000","gold_sense_keys":["audacious%5:00:00:adventurous:00"],"candidates":[{"index":1,"sense_key":"audacious%5:00:00:bold:00","synset_id":"audacious.s.01"},{"index":2,"sense_key":"audacious%5:00:00:unashamed:00","synset_id":"audacious.s.02"},{"index":3,"sense_key":"audacious%5:00:00:adventurous:00","synset_id":"audacious.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"audacious%5:00:00:adventurous:00","call_ids":["senseval2.d001.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"audacious%5:00:00:adventurous:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013445,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.997922207985539}
{"item_id":"senseval2.d001.s054.t001","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["senseval2.d001.s054.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021289999999999997,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0497112500015646}
{"item_id":"senseval2.d001.s055.t000","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"young%5:00:01:young:00","call_ids":["senseval2.d001.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"young%5:00:01:young:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021204999999999998,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.240390332997777}
{"item_id":"senseval2.d001.s055.t003","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval2.d001.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046835,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.252625584020279}
{"item_id":"senseval2.d001.s055.t004","gold_sense_keys":["right%3:00:04::"],"candidates":[{"index":1,"sense_key":"right%3:00:00::","synset_id":"right.a.01"},{"index":2,"sense_key":"right%3:00:02::","synset_id":"correct.a.01"},{"index":3,"sense_key":"right%5:00:01:proper:00","synset_id":"correct.s.01"},{"index":4,"sense_key":"right%3:00:01::","synset_id":"right.a.04"},{"index":5,"sense_key":"right%3:00:04::","synset_id":"right.a.05"},{"index":6,"sense_key":"right%5:00:00:appropriate:00","synset_id":"proper.s.03"},{"index":7,"sense_key":"right%3:00:03::","synset_id":"right.a.07"},{"index":8,"sense_key":"right%5:00:00:satisfactory:00","synset_id":"right.s.03"},{"index":9,"sense_key":"right%5:00:00:right-handed:00","synset_id":"right.s.04"},{"index":10,"sense_key":"right%5:00:02:proper:00","synset_id":"correct.s.02"},{"index":11,"sense_key":"right%5:00:00:perpendicular:00","synset_id":"right.s.06"},{"index":12,"sense_key":"right%5:00:00:outside:00","synset_id":"right.s.07"},{"index":13,"sense_key":"right%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":14,"sense_key":"right%5:00:00:accurate:00","synset_id":"veracious.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"right%3:00:02::","call_ids":["senseval2.d001.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"right%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":879,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035755,"input_uncached_usd":0.00004395,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.309268999961205}
{"item_id":"senseval2.d001.s055.t005","gold_sense_keys":["set_out%2:30:00::"],"candidates":[{"index":1,"sense_key":"set_out%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"set_out%2:35:00::","synset_id":"range.v.05"},{"index":3,"sense_key":"set_out%2:38:00::","synset_id":"depart.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set_out%2:30:00::","call_ids":["senseval2.d001.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set_out%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025985,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.16943683300633}
{"item_id":"senseval2.d001.s055.t006","gold_sense_keys":["repeat%2:36:00::"],"candidates":[{"index":1,"sense_key":"repeat%2:32:00::","synset_id":"repeat.v.01"},{"index":2,"sense_key":"repeat%2:36:00::","synset_id":"duplicate.v.01"},{"index":3,"sense_key":"repeat%2:30:00::","synset_id":"recur.v.01"},{"index":4,"sense_key":"repeat%2:32:01::","synset_id":"repeat.v.04"},{"index":5,"sense_key":"repeat%2:41:00::","synset_id":"repeat.v.05"},{"index":6,"sense_key":"repeat%2:36:01::","synset_id":"reprise.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"repeat%2:36:00::","call_ids":["senseval2.d001.s055.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"repeat%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003925,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.200160500011407}
{"item_id":"senseval2.d001.s055.t007","gold_sense_keys":["experiment%1:04:00::"],"candidates":[{"index":1,"sense_key":"experiment%1:04:00::","synset_id":"experiment.n.01"},{"index":2,"sense_key":"experiment%1:09:00::","synset_id":"experiment.n.02"},{"index":3,"sense_key":"experiment%1:04:01::","synset_id":"experiment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experiment%1:04:00::","call_ids":["senseval2.d001.s055.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experiment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023204999999999998,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.926210124976933}
{"item_id":"senseval2.d001.s055.t008","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cell%1:06:00::","call_ids":["senseval2.d001.s055.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cell%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006754,"input_uncached_usd":0.000029,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.599300290981773}
{"item_id":"senseval2.d001.s055.t009","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s055.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023524999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.605774291965645}
{"item_id":"senseval2.d001.s055.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s055.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013414999999999998,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.334094375022687}
{"item_id":"senseval2.d001.s056.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035829999999999993,"input_uncached_usd":0.0000191,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.314335458970163}
{"item_id":"senseval2.d001.s056.t002","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034264999999999995,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.029860624985304}
{"item_id":"senseval2.d001.s056.t003","gold_sense_keys":["dual%5:00:00:multiple:00"],"candidates":[{"index":1,"sense_key":"dual%5:00:00:multiple:00","synset_id":"double.s.02"},{"index":2,"sense_key":"dual%5:00:02:multiple:00","synset_id":"double.s.01"},{"index":3,"sense_key":"dual%5:00:00:plural:00","synset_id":"dual.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dual%5:00:00:multiple:00","call_ids":["senseval2.d001.s056.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dual%5:00:00:multiple:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003919,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.119149500038475}
{"item_id":"senseval2.d001.s056.t005","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s056.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047434999999999996,"input_uncached_usd":0.000032749999999999996,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.735554707993288}
{"item_id":"senseval2.d001.s056.t007","gold_sense_keys":["childhood%1:28:00::"],"candidates":[{"index":1,"sense_key":"childhood%1:28:00::","synset_id":"childhood.n.01"},{"index":2,"sense_key":"childhood%1:26:00::","synset_id":"childhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"childhood%1:28:00::","call_ids":["senseval2.d001.s056.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"childhood%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020495,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.991691584000364}
{"item_id":"senseval2.d001.s056.t008","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s056.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023754999999999997,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3608855420025066}
{"item_id":"senseval2.d001.s056.t010","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s056.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1504,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00033759999999999997,"input_uncached_usd":0.0000752,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4507547080283985}
{"item_id":"senseval2.d001.s057.t001","gold_sense_keys":["turn_to%2:30:00::"],"candidates":[{"index":1,"sense_key":"turn_to%2:32:01::","synset_id":"address.v.01"},{"index":2,"sense_key":"turn_to%2:30:00::","synset_id":"turn_to.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn_to%2:30:00::","call_ids":["senseval2.d001.s057.t001__v1__a1","senseval2.d001.s057.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn_to%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00067055,"input_uncached_usd":0.000043349999999999997,"input_cached_usd":0.0,"output_usd":0.0006272,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.151615916984156}
{"item_id":"senseval2.d001.s057.t005","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s057.t005__v1__a1","senseval2.d001.s057.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041485,"input_uncached_usd":0.000043650000000000004,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.888290542992763}
{"item_id":"senseval2.d001.s057.t006","gold_sense_keys":["big%3:00:01::","big%5:00:00:conspicuous:00"],"candidates":[{"index":1,"sense_key":"big%3:00:01::","synset_id":"large.a.01"},{"index":2,"sense_key":"big%5:00:00:important:00","synset_id":"big.s.01"},{"index":3,"sense_key":"big%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":4,"sense_key":"big%5:00:00:loud:00","synset_id":"big.s.03"},{"index":5,"sense_key":"big%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"big%5:00:00:intemperate:00","synset_id":"big.s.05"},{"index":7,"sense_key":"big%5:00:02:proud:00","synset_id":"boastful.s.01"},{"index":8,"sense_key":"big%5:00:00:proud:00","synset_id":"big.s.07"},{"index":9,"sense_key":"big%5:00:00:mature:01","synset_id":"adult.s.01"},{"index":10,"sense_key":"big%5:00:00:heavy:08","synset_id":"big.s.09"},{"index":11,"sense_key":"big%5:00:00:generous:02","synset_id":"big.s.10"},{"index":12,"sense_key":"big%5:00:00:generous:01","synset_id":"big.s.11"},{"index":13,"sense_key":"big%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"big%3:00:01::","call_ids":["senseval2.d001.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"big%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00023145,"input_uncached_usd":0.000045849999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.01845879200846}
{"item_id":"senseval2.d001.s057.t007","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s057.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021189999999999997,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9760637499857694}
{"item_id":"senseval2.d001.s057.t008","gold_sense_keys":["killer%1:26:00::"],"candidates":[{"index":1,"sense_key":"killer%1:18:00::","synset_id":"killer.n.01"},{"index":2,"sense_key":"killer%1:26:00::","synset_id":"cause_of_death.n.01"},{"index":3,"sense_key":"killer%1:09:00::","synset_id":"killer.n.03"},{"index":4,"sense_key":"killer%1:05:00::","synset_id":"killer_whale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"killer%1:26:00::","call_ids":["senseval2.d001.s057.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"killer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018424999999999998,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.922013125033118}
{"item_id":"senseval2.d001.s057.t009","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["senseval2.d001.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003358,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.959561333002057}
{"item_id":"senseval2.d001.s058.t000","gold_sense_keys":["believe%2:31:03::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"believe%2:31:04::","call_ids":["senseval2.d001.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"believe%2:31:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036734999999999996,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.7669822499738075}
{"item_id":"senseval2.d001.s058.t004","gold_sense_keys":["arise%2:42:00::"],"candidates":[{"index":1,"sense_key":"arise%2:42:00::","synset_id":"originate.v.01"},{"index":2,"sense_key":"arise%2:42:01::","synset_id":"arise.v.02"},{"index":3,"sense_key":"arise%2:38:00::","synset_id":"arise.v.03"},{"index":4,"sense_key":"arise%2:30:00::","synset_id":"arise.v.04"},{"index":5,"sense_key":"arise%2:38:04::","synset_id":"rise.v.01"},{"index":6,"sense_key":"arise%2:41:00::","synset_id":"rebel.v.01"},{"index":7,"sense_key":"arise%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"arise%2:38:00::","call_ids":["senseval2.d001.s058.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"arise%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005265,"input_uncached_usd":0.000033299999999999996,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.305910958035383}
{"item_id":"senseval2.d001.s058.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s058.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021279999999999997,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.756926625035703}
{"item_id":"senseval2.d001.s058.t008","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d001.s058.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002851,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7037518749712035}
{"item_id":"senseval2.d001.s058.t009","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval2.d001.s058.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026314999999999997,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.815832833002787}
{"item_id":"senseval2.d001.s058.t010","gold_sense_keys":["develop%2:30:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"develop%2:30:00::","call_ids":["senseval2.d001.s058.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"develop%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00042899999999999997,"input_uncached_usd":0.0000642,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.600323458958883}
{"item_id":"senseval2.d001.s058.t011","gold_sense_keys":["stage%1:26:00::","stage%1:28:00::"],"candidates":[{"index":1,"sense_key":"stage%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"stage%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"stage%1:06:00::","synset_id":"stage.n.03"},{"index":4,"sense_key":"stage%1:10:00::","synset_id":"stage.n.04"},{"index":5,"sense_key":"stage%1:06:01::","synset_id":"stagecoach.n.01"},{"index":6,"sense_key":"stage%1:04:00::","synset_id":"stage.n.06"},{"index":7,"sense_key":"stage%1:15:00::","synset_id":"stage.n.07"},{"index":8,"sense_key":"stage%1:06:02::","synset_id":"stage.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stage%1:26:00::","call_ids":["senseval2.d001.s058.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037539999999999996,"input_uncached_usd":0.0000362,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.450321417010855}
{"item_id":"senseval2.d001.s059.t000","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d001.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031125,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.1010205000056885}
{"item_id":"senseval2.d001.s059.t001","gold_sense_keys":["precede%2:42:00::"],"candidates":[{"index":1,"sense_key":"precede%2:42:00::","synset_id":"predate.v.01"},{"index":2,"sense_key":"precede%2:42:01::","synset_id":"precede.v.02"},{"index":3,"sense_key":"precede%2:41:00::","synset_id":"precede.v.03"},{"index":4,"sense_key":"precede%2:38:00::","synset_id":"precede.v.04"},{"index":5,"sense_key":"precede%2:32:03::","synset_id":"precede.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"precede%2:42:01::","call_ids":["senseval2.d001.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"precede%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":2960,"reasoning_output_tokens":2944},"cost":{"currency":"USD","total_usd":0.00121295,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.001184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.57782083301572}
{"item_id":"senseval2.d001.s059.t002","gold_sense_keys":["development%1:22:01::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032955,"input_uncached_usd":0.00004155,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.161264875030611}
{"item_id":"senseval2.d001.s059.t003","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001582,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9401955000357702}
{"item_id":"senseval2.d001.s059.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["senseval2.d001.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1216,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003232,"input_uncached_usd":0.0000608,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.116275624954142}
{"item_id":"senseval2.d001.s059.t006","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"become%2:42:01::","call_ids":["senseval2.d001.s059.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"become%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044195,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.994362582976464}
{"item_id":"senseval2.d001.s059.t010","gold_sense_keys":["stage%1:26:00::","stage%1:28:00::"],"candidates":[{"index":1,"sense_key":"stage%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"stage%1:26:00::","synset_id":"degree.n.02"},{"index":3,"sense_key":"stage%1:06:00::","synset_id":"stage.n.03"},{"index":4,"sense_key":"stage%1:10:00::","synset_id":"stage.n.04"},{"index":5,"sense_key":"stage%1:06:01::","synset_id":"stagecoach.n.01"},{"index":6,"sense_key":"stage%1:04:00::","synset_id":"stage.n.06"},{"index":7,"sense_key":"stage%1:15:00::","synset_id":"stage.n.07"},{"index":8,"sense_key":"stage%1:06:02::","synset_id":"stage.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stage%1:26:00::","call_ids":["senseval2.d001.s059.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stage%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003759999999999999,"input_uncached_usd":0.0000368,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.8491097500082105}
{"item_id":"senseval2.d001.s059.t011","gold_sense_keys":["progress%2:38:00::"],"candidates":[{"index":1,"sense_key":"progress%2:30:00::","synset_id":"progress.v.01"},{"index":2,"sense_key":"progress%2:38:00::","synset_id":"advance.v.01"},{"index":3,"sense_key":"progress%2:30:01::","synset_id":"build_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"progress%2:38:00::","call_ids":["senseval2.d001.s059.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"progress%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008512,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.38676566700451}
{"item_id":"senseval2.d001.s059.t012","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d001.s059.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028405,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.728847291029524}
{"item_id":"senseval2.d001.s059.t013","gold_sense_keys":["severe%5:00:00:critical:03","severe%5:00:00:intense:00","severe%5:00:01:bad:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":2,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":3,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":4,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":5,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":6,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"severe%5:00:01:bad:00","call_ids":["senseval2.d001.s059.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"severe%5:00:01:bad:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003715,"input_uncached_usd":0.0000323,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.339557125000283}
{"item_id":"senseval2.d001.s059.t014","gold_sense_keys":["deadly%5:00:01:fatal:00"],"candidates":[{"index":1,"sense_key":"deadly%5:00:01:fatal:00","synset_id":"deadly.s.01"},{"index":2,"sense_key":"deadly%5:00:02:fatal:00","synset_id":"deadly.s.02"},{"index":3,"sense_key":"deadly%5:00:00:toxic:00","synset_id":"deadly.s.03"},{"index":4,"sense_key":"deadly%5:00:00:unpardonable:00","synset_id":"deadly.s.04"},{"index":5,"sense_key":"deadly%5:00:00:noxious:00","synset_id":"baneful.s.01"},{"index":6,"sense_key":"deadly%5:00:00:virulent:00","synset_id":"deadly.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"deadly%5:00:01:fatal:00","call_ids":["senseval2.d001.s059.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"deadly%5:00:01:fatal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044635,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.786481208982877}
{"item_id":"senseval2.d001.s059.t015","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s059.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039145,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.836221416015178}
{"item_id":"senseval2.d001.s059.t016","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s059.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":2448,"reasoning_output_tokens":2432},"cost":{"currency":"USD","total_usd":0.00100555,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0009792,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.758194915950298}
{"item_id":"senseval2.d001.s059.t017","gold_sense_keys":["occur%2:30:00::"],"candidates":[{"index":1,"sense_key":"occur%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"occur%2:31:00::","synset_id":"occur.v.02"},{"index":3,"sense_key":"occur%2:42:03::","synset_id":"occur.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"occur%2:30:00::","call_ids":["senseval2.d001.s059.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"occur%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018485,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8979914999799803}
{"item_id":"senseval2.d001.s060.t001","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d001.s060.t001__v1__a1","senseval2.d001.s060.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":737,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00034324999999999997,"input_uncached_usd":0.00004845,"input_cached_usd":0.0,"output_usd":0.0002948,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.725560249004047}
{"item_id":"senseval2.d001.s060.t002","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005885,"input_uncached_usd":0.0000445,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.984529250010382}
{"item_id":"senseval2.d001.s060.t003","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s060.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020845,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.281550707994029}
{"item_id":"senseval2.d001.s060.t004","gold_sense_keys":["tedious%5:00:00:uninteresting:00"],"candidates":[{"index":1,"sense_key":"tedious%5:00:00:uninteresting:00","synset_id":"boring.s.01"},{"index":2,"sense_key":"tedious%5:00:00:prolix:00","synset_id":"long-winded.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tedious%5:00:00:uninteresting:00","call_ids":["senseval2.d001.s060.t004__v1__a1","senseval2.d001.s060.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tedious%5:00:00:uninteresting:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1097,"cached_input_tokens":0,"output_tokens":1122,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00050365,"input_uncached_usd":0.00005484999999999999,"input_cached_usd":0.0,"output_usd":0.00044879999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.956920749973506}
{"item_id":"senseval2.d001.s060.t005","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"often%4:02:02::","call_ids":["senseval2.d001.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"often%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005174,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.870557541027665}
{"item_id":"senseval2.d001.s060.t009","gold_sense_keys":["search%2:35:00::","search%2:39:00::"],"candidates":[{"index":1,"sense_key":"search%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"search%2:39:00::","synset_id":"search.v.02"},{"index":3,"sense_key":"search%2:31:00::","synset_id":"research.v.02"},{"index":4,"sense_key":"search%2:35:01::","synset_id":"search.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"search%2:35:00::","call_ids":["senseval2.d001.s060.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"search%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005474,"input_uncached_usd":0.000029,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.949458874994889}
{"item_id":"senseval2.d001.s060.t010","gold_sense_keys":["sign%1:26:00::"],"candidates":[{"index":1,"sense_key":"sign%1:10:05::","synset_id":"sign.n.01"},{"index":2,"sense_key":"sign%1:10:00::","synset_id":"sign.n.02"},{"index":3,"sense_key":"sign%1:10:03::","synset_id":"signal.n.01"},{"index":4,"sense_key":"sign%1:06:01::","synset_id":"signboard.n.01"},{"index":5,"sense_key":"sign%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":6,"sense_key":"sign%1:26:00::","synset_id":"sign.n.06"},{"index":7,"sense_key":"sign%1:24:00::","synset_id":"polarity.n.02"},{"index":8,"sense_key":"sign%1:11:00::","synset_id":"augury.n.01"},{"index":9,"sense_key":"sign%1:10:01::","synset_id":"sign.n.09"},{"index":10,"sense_key":"sign%1:10:06::","synset_id":"sign.n.10"},{"index":11,"sense_key":"sign%1:10:04::","synset_id":"sign.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sign%1:10:05::","call_ids":["senseval2.d001.s060.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sign%1:10:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":863,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035674999999999997,"input_uncached_usd":0.00004315,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.018653750012163}
{"item_id":"senseval2.d001.s060.t011","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s060.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002648,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.148338916013017}
{"item_id":"senseval2.d001.s060.t012","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["senseval2.d001.s060.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049485,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.002231708960608}
{"item_id":"senseval2.d001.s061.t000","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval2.d001.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048444999999999993,"input_uncached_usd":0.00004285,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.039695457962807}
{"item_id":"senseval2.d001.s061.t001","gold_sense_keys":["confusing%5:00:00:unclear:00"],"candidates":[{"index":1,"sense_key":"confusing%5:00:00:disorienting:00","synset_id":"confusing.s.01"},{"index":2,"sense_key":"confusing%5:00:00:unclear:00","synset_id":"confusing.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confusing%5:00:00:unclear:00","call_ids":["senseval2.d001.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confusing%5:00:00:unclear:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023414999999999997,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1629508329788223}
{"item_id":"senseval2.d001.s061.t002","gold_sense_keys":["variety%1:07:00::","variety%1:14:01::"],"candidates":[{"index":1,"sense_key":"variety%1:14:01::","synset_id":"assortment.n.01"},{"index":2,"sense_key":"variety%1:07:00::","synset_id":"diverseness.n.01"},{"index":3,"sense_key":"variety%1:14:00::","synset_id":"variety.n.03"},{"index":4,"sense_key":"variety%1:10:00::","synset_id":"variety_show.n.01"},{"index":5,"sense_key":"variety%1:09:01::","synset_id":"kind.n.01"},{"index":6,"sense_key":"variety%1:07:01::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"variety%1:07:00::","call_ids":["senseval2.d001.s061.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"variety%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003206,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.052641333953943}
{"item_id":"senseval2.d001.s061.t003","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s061.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033995,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.239053208031692}
{"item_id":"senseval2.d001.s061.t004","gold_sense_keys":["exist%2:42:00::"],"candidates":[{"index":1,"sense_key":"exist%2:42:00::","synset_id":"exist.v.01"},{"index":2,"sense_key":"exist%2:42:01::","synset_id":"exist.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exist%2:42:00::","call_ids":["senseval2.d001.s061.t004__v1__a1","senseval2.d001.s061.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exist%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005197499999999999,"input_uncached_usd":0.00004615,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.76110629196046}
{"item_id":"senseval2.d001.s061.t005","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d001.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003962,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.727595667005517}
{"item_id":"senseval2.d001.s061.t006","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s061.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059305,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.874742957996204}
{"item_id":"senseval2.d001.s061.t008","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029385,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.374552958004642}
{"item_id":"senseval2.d001.s061.t010","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s061.t010__v1__a1","senseval2.d001.s061.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":987,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004973499999999999,"input_uncached_usd":0.00004934999999999999,"input_cached_usd":0.0,"output_usd":0.000448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.983356542012189}
{"item_id":"senseval2.d001.s061.t012","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s061.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047259999999999994,"input_uncached_usd":0.000031,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.62260670796968}
{"item_id":"senseval2.d001.s062.t001","gold_sense_keys":["coherent%3:00:00::"],"candidates":[{"index":1,"sense_key":"coherent%3:00:00::","synset_id":"coherent.a.01"},{"index":2,"sense_key":"coherent%5:00:00:rational:00","synset_id":"coherent.s.01"},{"index":3,"sense_key":"coherent%3:00:01::","synset_id":"coherent.a.03"},{"index":4,"sense_key":"coherent%5:00:00:adhesive:00","synset_id":"coherent.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coherent%3:00:00::","call_ids":["senseval2.d001.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coherent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021099999999999998,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6870899999630637}
{"item_id":"senseval2.d001.s062.t002","gold_sense_keys":["picture%1:26:00::"],"candidates":[{"index":1,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":2,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":3,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":4,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":5,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":6,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":7,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":8,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":9,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":10,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"picture%1:09:00::","call_ids":["senseval2.d001.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"picture%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017795,"input_uncached_usd":0.000043549999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.670985542004928}
{"item_id":"senseval2.d001.s062.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003660499999999999,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.351431875023991}
{"item_id":"senseval2.d001.s062.t004","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s062.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040544999999999996,"input_uncached_usd":0.00004065,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.960273124976084}
{"item_id":"senseval2.d001.s062.t005","gold_sense_keys":["emerge%2:30:01::"],"candidates":[{"index":1,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":2,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":3,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":4,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":5,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emerge%2:30:01::","call_ids":["senseval2.d001.s062.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emerge%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0000585,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.000032,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.1697507500066422}
{"item_id":"senseval2.d001.s063.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023524999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.897255417017732}
{"item_id":"senseval2.d001.s063.t001","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021825,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.922625417006202}
{"item_id":"senseval2.d001.s063.t003","gold_sense_keys":["knock_out%2:30:01::"],"candidates":[{"index":1,"sense_key":"knock_out%2:30:01::","synset_id":"knock_out.v.01"},{"index":2,"sense_key":"knock_out%2:35:00::","synset_id":"knock_cold.v.01"},{"index":3,"sense_key":"knock_out%2:30:00::","synset_id":"knock_out.v.03"},{"index":4,"sense_key":"knock_out%2:37:00::","synset_id":"knock_out.v.04"},{"index":5,"sense_key":"knock_out%2:30:02::","synset_id":"knock_out.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"knock_out%2:30:01::","call_ids":["senseval2.d001.s063.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"knock_out%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021124999999999998,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3159667089930736}
{"item_id":"senseval2.d001.s063.t004","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018265,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9944422079715878}
{"item_id":"senseval2.d001.s063.t005","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"develop%2:29:00::","call_ids":["senseval2.d001.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"develop%2:29:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1271,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00053115,"input_uncached_usd":0.00006355,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.310458166000899}
{"item_id":"senseval2.d001.s064.t000","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002348,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.938113957992755}
{"item_id":"senseval2.d001.s064.t001","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval2.d001.s064.t001__v1__a1","senseval2.d001.s064.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038894999999999994,"input_uncached_usd":0.000043349999999999997,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9452377909910865}
{"item_id":"senseval2.d001.s064.t003","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"then%4:02:01::","call_ids":["senseval2.d001.s064.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"then%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043854999999999995,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.391502082988154}
{"item_id":"senseval2.d001.s064.t004","gold_sense_keys":["delete%2:35:00::"],"candidates":[{"index":1,"sense_key":"delete%2:35:00::","synset_id":"delete.v.01"},{"index":2,"sense_key":"delete%2:32:00::","synset_id":"erase.v.03"},{"index":3,"sense_key":"delete%2:30:00::","synset_id":"edit.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"delete%2:30:00::","call_ids":["senseval2.d001.s064.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"delete%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033519999999999996,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.176688082981855}
{"item_id":"senseval2.d001.s064.t005","gold_sense_keys":["polyp%1:26:00::"],"candidates":[{"index":1,"sense_key":"polyp%1:26:00::","synset_id":"polyp.n.01"},{"index":2,"sense_key":"polyp%1:05:00::","synset_id":"polyp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"polyp%1:26:00::","call_ids":["senseval2.d001.s064.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"polyp%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002082,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4119047499843873}
{"item_id":"senseval2.d001.s064.t007","gold_sense_keys":["malignancy%1:26:00::"],"candidates":[{"index":1,"sense_key":"malignancy%1:26:00::","synset_id":"malignancy.n.01"},{"index":2,"sense_key":"malignancy%1:07:00::","synset_id":"malignity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"malignancy%1:26:00::","call_ids":["senseval2.d001.s064.t007__v1__a1","senseval2.d001.s064.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"malignancy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046504999999999995,"input_uncached_usd":0.00004264999999999999,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.570792250044178}
{"item_id":"senseval2.d001.s065.t000","gold_sense_keys":["clear%3:00:00::"],"candidates":[{"index":1,"sense_key":"clear%3:00:00::","synset_id":"clear.a.01"},{"index":2,"sense_key":"clear%5:00:00:clearheaded:00","synset_id":"clear.s.01"},{"index":3,"sense_key":"clear%5:00:00:unobstructed:00","synset_id":"clear.s.02"},{"index":4,"sense_key":"clear%3:00:02::","synset_id":"clear.a.04"},{"index":5,"sense_key":"clear%5:00:02:free:00","synset_id":"clear.s.03"},{"index":6,"sense_key":"clear%5:00:00:untroubled:00","synset_id":"clear.s.04"},{"index":7,"sense_key":"clear%5:00:01:pure:02","synset_id":"clean.s.02"},{"index":8,"sense_key":"clear%5:00:00:unencumbered:00","synset_id":"clear.s.06"},{"index":9,"sense_key":"clear%5:00:00:distinct:00","synset_id":"clear.s.07"},{"index":10,"sense_key":"clear%3:00:04::","synset_id":"well-defined.a.02"},{"index":11,"sense_key":"clear%3:00:03::","synset_id":"clear.a.11"},{"index":12,"sense_key":"clear%5:00:00:unqualified:02","synset_id":"clean.s.01"},{"index":13,"sense_key":"clear%5:00:00:perfect:00","synset_id":"clear.s.09"},{"index":14,"sense_key":"clear%5:00:00:net:00","synset_id":"clear.s.10"},{"index":15,"sense_key":"clear%5:00:00:legible:00","synset_id":"clear.s.11"},{"index":16,"sense_key":"clear%5:00:00:innocent:00","synset_id":"absolved.s.01"},{"index":17,"sense_key":"clear%5:00:00:discerning:00","synset_id":"clear.s.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"clear%3:00:00::","call_ids":["senseval2.d001.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"clear%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00028865,"input_uncached_usd":0.00005185,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.414791207993403}
{"item_id":"senseval2.d001.s065.t003","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%2:30:01::","call_ids":["senseval2.d001.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051075,"input_uncached_usd":0.00001795,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.291238999983761}
{"item_id":"senseval2.d001.s065.t006","gold_sense_keys":["develop%2:30:00::","develop%2:42:00::"],"candidates":[{"index":1,"sense_key":"develop%2:36:01::","synset_id":"develop.v.01"},{"index":2,"sense_key":"develop%2:36:00::","synset_id":"evolve.v.01"},{"index":3,"sense_key":"develop%2:30:06::","synset_id":"develop.v.03"},{"index":4,"sense_key":"develop%2:29:00::","synset_id":"grow.v.08"},{"index":5,"sense_key":"develop%2:42:00::","synset_id":"originate.v.01"},{"index":6,"sense_key":"develop%2:30:04::","synset_id":"build_up.v.05"},{"index":7,"sense_key":"develop%2:32:00::","synset_id":"explicate.v.02"},{"index":8,"sense_key":"develop%2:31:00::","synset_id":"train.v.01"},{"index":9,"sense_key":"develop%2:30:05::","synset_id":"develop.v.09"},{"index":10,"sense_key":"develop%2:30:00::","synset_id":"develop.v.10"},{"index":11,"sense_key":"develop%2:30:09::","synset_id":"modernize.v.02"},{"index":12,"sense_key":"develop%2:30:01::","synset_id":"develop.v.12"},{"index":13,"sense_key":"develop%2:36:09::","synset_id":"develop.v.13"},{"index":14,"sense_key":"develop%2:30:10::","synset_id":"develop.v.14"},{"index":15,"sense_key":"develop%2:39:00::","synset_id":"develop.v.15"},{"index":16,"sense_key":"develop%2:35:04::","synset_id":"develop.v.16"},{"index":17,"sense_key":"develop%2:33:03::","synset_id":"develop.v.17"},{"index":18,"sense_key":"develop%2:33:02::","synset_id":"develop.v.18"},{"index":19,"sense_key":"develop%2:30:13::","synset_id":"develop.v.19"},{"index":20,"sense_key":"develop%2:30:03::","synset_id":"break.v.53"},{"index":21,"sense_key":"develop%2:30:12::","synset_id":"develop.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"develop%2:30:00::","call_ids":["senseval2.d001.s065.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"develop%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1227,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047735,"input_uncached_usd":0.00006135,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.3807110419729725}
{"item_id":"senseval2.d001.s066.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021649999999999998,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8327544589992613}
{"item_id":"senseval2.d001.s066.t001","gold_sense_keys":["galvanize%2:37:00::"],"candidates":[{"index":1,"sense_key":"galvanize%2:37:00::","synset_id":"startle.v.01"},{"index":2,"sense_key":"galvanize%2:35:00::","synset_id":"galvanize.v.02"},{"index":3,"sense_key":"galvanize%2:29:02::","synset_id":"galvanize.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"galvanize%2:37:00::","call_ids":["senseval2.d001.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"galvanize%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002316,"input_uncached_usd":0.000020399999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.878620375005994}
{"item_id":"senseval2.d001.s066.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018334999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7106945830164477}
{"item_id":"senseval2.d001.s067.t000","gold_sense_keys":["confirming%5:00:00:supportive:00"],"candidates":[{"index":1,"sense_key":"confirming%3:00:02::","synset_id":"positive.a.04"},{"index":2,"sense_key":"confirming%5:00:00:supportive:00","synset_id":"collateral.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confirming%5:00:00:supportive:00","call_ids":["senseval2.d001.s067.t000__v1__a1","senseval2.d001.s067.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confirming%5:00:00:supportive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":1633,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00069405,"input_uncached_usd":0.000040850000000000004,"input_cached_usd":0.0,"output_usd":0.0006531999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.187205957947299}
{"item_id":"senseval2.d001.s067.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d001.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018054999999999997,"input_uncached_usd":0.000020549999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.630149665987119}
{"item_id":"senseval2.d001.s067.t002","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d001.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"need%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028314999999999997,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.496208500000648}
{"item_id":"senseval2.d001.s067.t004","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["senseval2.d001.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003189,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3545804169843905}
{"item_id":"senseval2.d001.s067.t005","gold_sense_keys":["critical%5:00:00:indispensable:00"],"candidates":[{"index":1,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":2,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":3,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":4,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":5,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":6,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":7,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"critical%5:00:00:indispensable:00","call_ids":["senseval2.d001.s067.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"critical%5:00:00:indispensable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024055,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.426784041977953}
{"item_id":"senseval2.d001.s067.t006","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d001.s067.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003263,"input_uncached_usd":0.000038299999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.712082209007349}
{"item_id":"senseval2.d001.s067.t007","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"common%3:00:01::","call_ids":["senseval2.d001.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"common%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005293,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.57154533296125}
{"item_id":"senseval2.d001.s067.t009","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004033,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.711723166983575}
{"item_id":"senseval2.d001.s068.t000","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":2,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":3,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":4,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"},{"index":5,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":6,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"yet%4:02:04::","call_ids":["senseval2.d001.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"yet%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024244999999999998,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.689731457969174}
{"item_id":"senseval2.d001.s068.t001","gold_sense_keys":["nail%2:40:00::"],"candidates":[{"index":1,"sense_key":"nail%2:35:00::","synset_id":"nail.v.01"},{"index":2,"sense_key":"nail%2:35:01::","synset_id":"collar.v.01"},{"index":3,"sense_key":"nail%2:35:02::","synset_id":"smash.v.01"},{"index":4,"sense_key":"nail%2:41:01::","synset_id":"nail_down.v.01"},{"index":5,"sense_key":"nail%2:41:00::","synset_id":"breeze_through.v.01"},{"index":6,"sense_key":"nail%2:40:00::","synset_id":"pinpoint.v.01"},{"index":7,"sense_key":"nail%2:33:00::","synset_id":"complete.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"nail%2:40:00::","call_ids":["senseval2.d001.s068.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"nail%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021404999999999997,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.951798416033853}
{"item_id":"senseval2.d001.s068.t002","gold_sense_keys":["identity%1:09:00::"],"candidates":[{"index":1,"sense_key":"identity%1:07:00::","synset_id":"identity.n.01"},{"index":2,"sense_key":"identity%1:09:00::","synset_id":"identity.n.02"},{"index":3,"sense_key":"identity%1:24:01::","synset_id":"identity.n.03"},{"index":4,"sense_key":"identity%1:07:02::","synset_id":"identity.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"identity%1:09:00::","call_ids":["senseval2.d001.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"identity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018385,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.470289667020552}
{"item_id":"senseval2.d001.s068.t004","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%2:30:01::","call_ids":["senseval2.d001.s068.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048499999999999997,"input_uncached_usd":0.0000178,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.437397208006587}
{"item_id":"senseval2.d001.s068.t005","gold_sense_keys":["flip%2:30:00::"],"candidates":[{"index":1,"sense_key":"flip%2:38:00::","synset_id":"flip.v.01"},{"index":2,"sense_key":"flip%2:35:01::","synset_id":"throw.v.06"},{"index":3,"sense_key":"flip%2:39:10::","synset_id":"flick.v.02"},{"index":4,"sense_key":"flip%2:38:02::","synset_id":"flip.v.04"},{"index":5,"sense_key":"flip%2:38:01::","synset_id":"flip.v.05"},{"index":6,"sense_key":"flip%2:35:00::","synset_id":"flip.v.06"},{"index":7,"sense_key":"flip%2:35:05::","synset_id":"flip.v.07"},{"index":8,"sense_key":"flip%2:35:03::","synset_id":"flip.v.08"},{"index":9,"sense_key":"flip%2:31:00::","synset_id":"flip.v.09"},{"index":10,"sense_key":"flip%2:30:13::","synset_id":"flip.v.10"},{"index":11,"sense_key":"flip%2:30:00::","synset_id":"interchange.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"flip%2:30:00::","call_ids":["senseval2.d001.s068.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"flip%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.000428,"input_uncached_usd":0.0000376,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.212774207990151}
{"item_id":"senseval2.d001.s068.t006","gold_sense_keys":["colon%1:08:00::"],"candidates":[{"index":1,"sense_key":"colon%1:08:00::","synset_id":"colon.n.01"},{"index":2,"sense_key":"colon%1:23:01::","synset_id":"colon.n.02"},{"index":3,"sense_key":"colon%1:23:00::","synset_id":"colon.n.03"},{"index":4,"sense_key":"colon%1:15:00::","synset_id":"colon.n.04"},{"index":5,"sense_key":"colon%1:10:00::","synset_id":"colon.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"colon%1:08:00::","call_ids":["senseval2.d001.s068.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"colon%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026095,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.54868195799645}
{"item_id":"senseval2.d001.s068.t007","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s068.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004955,"input_uncached_usd":0.000027899999999999997,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.691655249975156}
{"item_id":"senseval2.d001.s068.t008","gold_sense_keys":["full-blown%5:00:00:complete:00"],"candidates":[{"index":1,"sense_key":"full-blown%5:00:00:mature:01","synset_id":"full-blown.s.01"},{"index":2,"sense_key":"full-blown%5:00:00:complete:00","synset_id":"full-blown.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"full-blown%5:00:00:complete:00","call_ids":["senseval2.d001.s068.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"full-blown%5:00:00:complete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005369,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.699608000053558}
{"item_id":"senseval2.d001.s068.t009","gold_sense_keys":["malignancy%1:26:00::"],"candidates":[{"index":1,"sense_key":"malignancy%1:26:00::","synset_id":"malignancy.n.01"},{"index":2,"sense_key":"malignancy%1:07:00::","synset_id":"malignity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"malignancy%1:26:00::","call_ids":["senseval2.d001.s068.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"malignancy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030670000000000003,"input_uncached_usd":0.000018299999999999998,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.048700249986723}
{"item_id":"senseval2.d001.s070.t000","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["senseval2.d001.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004862,"input_uncached_usd":0.000018599999999999998,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.511254707991611}
{"item_id":"senseval2.d001.s070.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d001.s070.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003425,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.528051666973624}
{"item_id":"senseval2.d001.s070.t005","gold_sense_keys":["down%4:02:00::"],"candidates":[{"index":1,"sense_key":"down%4:02:00::","synset_id":"down.r.01"},{"index":2,"sense_key":"down%4:02:02::","synset_id":"down.r.02"},{"index":3,"sense_key":"down%4:02:03::","synset_id":"down.r.03"},{"index":4,"sense_key":"down%4:02:06::","synset_id":"down.r.04"},{"index":5,"sense_key":"down%4:02:05::","synset_id":"down.r.05"},{"index":6,"sense_key":"down%4:02:04::","synset_id":"down.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"down%4:02:00::","call_ids":["senseval2.d001.s070.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"down%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003151,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.878443958994467}
{"item_id":"senseval2.d001.s070.t006","gold_sense_keys":["length%1:07:00::"],"candidates":[{"index":1,"sense_key":"length%1:07:00::","synset_id":"length.n.01"},{"index":2,"sense_key":"length%1:07:01::","synset_id":"duration.n.03"},{"index":3,"sense_key":"length%1:07:02::","synset_id":"length.n.03"},{"index":4,"sense_key":"length%1:07:03::","synset_id":"distance.n.03"},{"index":5,"sense_key":"length%1:06:00::","synset_id":"length.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"length%1:07:00::","call_ids":["senseval2.d001.s070.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"length%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041685,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.543315874994732}
{"item_id":"senseval2.d001.s070.t008","gold_sense_keys":["look_for%2:35:00::"],"candidates":[{"index":1,"sense_key":"look_for%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"look_for%2:37:00::","synset_id":"anticipate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look_for%2:35:00::","call_ids":["senseval2.d001.s070.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look_for%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038415,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.405706791963894}
{"item_id":"senseval2.d001.s070.t009","gold_sense_keys":["common%5:00:00:shared:00"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"common%5:00:00:shared:00","call_ids":["senseval2.d001.s070.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"common%5:00:00:shared:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024845,"input_uncached_usd":0.00003685,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3914179999846965}
{"item_id":"senseval2.d001.s070.t010","gold_sense_keys":["bit%1:17:00::","bit%1:23:01::"],"candidates":[{"index":1,"sense_key":"bit%1:23:01::","synset_id":"spot.n.10"},{"index":2,"sense_key":"bit%1:17:00::","synset_id":"bit.n.02"},{"index":3,"sense_key":"bit%1:28:00::","synset_id":"moment.n.02"},{"index":4,"sense_key":"bit%1:11:00::","synset_id":"piece.n.05"},{"index":5,"sense_key":"bit%1:06:01::","synset_id":"bit.n.05"},{"index":6,"sense_key":"bit%1:23:00::","synset_id":"bit.n.06"},{"index":7,"sense_key":"bit%1:13:00::","synset_id":"morsel.n.02"},{"index":8,"sense_key":"bit%1:10:01::","synset_id":"snatch.n.01"},{"index":9,"sense_key":"bit%1:10:00::","synset_id":"act.n.04"},{"index":10,"sense_key":"bit%1:06:02::","synset_id":"bit.n.10"},{"index":11,"sense_key":"bit%1:06:00::","synset_id":"bit.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bit%1:17:00::","call_ids":["senseval2.d001.s070.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bit%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045519999999999995,"input_uncached_usd":0.0000392,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.954541624989361}
{"item_id":"senseval2.d001.s070.t011","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["senseval2.d001.s070.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041455,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.384127958037425}
{"item_id":"senseval2.d001.s070.t012","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":2,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":3,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":4,"sense_key":"material%1:06:01::","synset_id":"material.n.04"},{"index":5,"sense_key":"material%1:18:00::","synset_id":"material.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"material%1:27:00::","call_ids":["senseval2.d001.s070.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004172,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.214328457950614}
{"item_id":"senseval2.d001.s070.t013","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval2.d001.s070.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003772499999999999,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.341598000028171}
{"item_id":"senseval2.d001.s070.t015","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s070.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002652,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.108693707967177}
{"item_id":"senseval2.d001.s071.t000","gold_sense_keys":["piece%1:06:00::","piece%1:17:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"piece%1:17:00::","call_ids":["senseval2.d001.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"piece%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035644999999999996,"input_uncached_usd":0.00004285,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.060562249971554}
{"item_id":"senseval2.d001.s071.t002","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"probably%4:02:01::","call_ids":["senseval2.d001.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"probably%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020605,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.411516708030831}
{"item_id":"senseval2.d001.s071.t003","gold_sense_keys":["constitute%2:42:00::"],"candidates":[{"index":1,"sense_key":"constitute%2:42:00::","synset_id":"constitute.v.01"},{"index":2,"sense_key":"constitute%2:41:00::","synset_id":"appoint.v.01"},{"index":3,"sense_key":"constitute%2:42:03::","synset_id":"form.v.02"},{"index":4,"sense_key":"constitute%2:36:00::","synset_id":"establish.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constitute%2:42:00::","call_ids":["senseval2.d001.s071.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constitute%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054225,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.598468750016764}
{"item_id":"senseval2.d001.s072.t000","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["senseval2.d001.s072.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1120,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009327999999999999,"input_uncached_usd":0.000056,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.375542958034202}
{"item_id":"senseval2.d001.s072.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024825,"input_uncached_usd":0.00003705,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.094875958980992}
{"item_id":"senseval2.d001.s072.t003","gold_sense_keys":["dubious%5:00:00:uncertain:02","dubious%5:00:00:unconvinced:00"],"candidates":[{"index":1,"sense_key":"dubious%5:00:00:uncertain:02","synset_id":"doubtful.s.02"},{"index":2,"sense_key":"dubious%5:00:00:questionable:00","synset_id":"doubtful.s.01"},{"index":3,"sense_key":"dubious%5:00:00:unconvinced:00","synset_id":"dubious.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dubious%5:00:00:unconvinced:00","call_ids":["senseval2.d001.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dubious%5:00:00:unconvinced:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018404999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4926192500279285}
{"item_id":"senseval2.d001.s072.t004","gold_sense_keys":["search%1:04:00::","search%1:04:02::"],"candidates":[{"index":1,"sense_key":"search%1:04:00::","synset_id":"search.n.01"},{"index":2,"sense_key":"search%1:04:02::","synset_id":"search.n.02"},{"index":3,"sense_key":"search%1:22:00::","synset_id":"search.n.03"},{"index":4,"sense_key":"search%1:09:00::","synset_id":"search.n.04"},{"index":5,"sense_key":"search%1:04:01::","synset_id":"search.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"search%1:22:00::","call_ids":["senseval2.d001.s072.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"search%1:22:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009027999999999999,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.874379542015959}
{"item_id":"senseval2.d001.s073.t000","gold_sense_keys":["doubt%1:07:00::"],"candidates":[{"index":1,"sense_key":"doubt%1:09:00::","synset_id":"doubt.n.01"},{"index":2,"sense_key":"doubt%1:07:00::","synset_id":"doubt.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doubt%1:09:00::","call_ids":["senseval2.d001.s073.t000__v1__a1","senseval2.d001.s073.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doubt%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":875,"cached_input_tokens":0,"output_tokens":1888,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007989499999999999,"input_uncached_usd":0.00004374999999999999,"input_cached_usd":0.0,"output_usd":0.0007551999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.429441499931272}
{"item_id":"senseval2.d001.s073.t001","gold_sense_keys":["stem%2:42:00::"],"candidates":[{"index":1,"sense_key":"stem%2:42:00::","synset_id":"stem.v.01"},{"index":2,"sense_key":"stem%2:42:03::","synset_id":"stem.v.02"},{"index":3,"sense_key":"stem%2:30:00::","synset_id":"stem.v.03"},{"index":4,"sense_key":"stem%2:30:01::","synset_id":"stem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stem%2:42:00::","call_ids":["senseval2.d001.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020854999999999997,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.26929874997586}
{"item_id":"senseval2.d001.s073.t002","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fact%1:10:01::","call_ids":["senseval2.d001.s073.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fact%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005937500000000001,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.62729616696015}
{"item_id":"senseval2.d001.s073.t003","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["senseval2.d001.s073.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018204999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.66460974997608}
{"item_id":"senseval2.d001.s073.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d001.s073.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016005,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.753159957996104}
{"item_id":"senseval2.d001.s073.t005","gold_sense_keys":["earlier%4:02:00::"],"candidates":[{"index":1,"sense_key":"earlier%4:02:00::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"earlier%4:02:02::","synset_id":"sooner.r.01"},{"index":3,"sense_key":"earlier%4:02:04::","synset_id":"in_the_first_place.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"earlier%4:02:04::","call_ids":["senseval2.d001.s073.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"earlier%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005152500000000001,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.722363082983065}
{"item_id":"senseval2.d001.s073.t007","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"find%2:40:00::","call_ids":["senseval2.d001.s073.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"find%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1101,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00067585,"input_uncached_usd":0.000055049999999999996,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.95018004195299}
{"item_id":"senseval2.d001.s073.t008","gold_sense_keys":["experiment%1:04:00::"],"candidates":[{"index":1,"sense_key":"experiment%1:04:00::","synset_id":"experiment.n.01"},{"index":2,"sense_key":"experiment%1:09:00::","synset_id":"experiment.n.02"},{"index":3,"sense_key":"experiment%1:04:01::","synset_id":"experiment.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experiment%1:04:00::","call_ids":["senseval2.d001.s073.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experiment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020659999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2453135409741662}
{"item_id":"senseval2.d001.s073.t009","gold_sense_keys":["mouse%1:05:00::"],"candidates":[{"index":1,"sense_key":"mouse%1:05:00::","synset_id":"mouse.n.01"},{"index":2,"sense_key":"mouse%1:26:00::","synset_id":"shiner.n.01"},{"index":3,"sense_key":"mouse%1:18:00::","synset_id":"mouse.n.03"},{"index":4,"sense_key":"mouse%1:06:00::","synset_id":"mouse.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mouse%1:05:00::","call_ids":["senseval2.d001.s073.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mouse%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018429999999999998,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0135434590047225}
{"item_id":"senseval2.d001.s073.t011","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d001.s073.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1487,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00020874999999999998,"input_uncached_usd":0.00007434999999999999,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.404028541990556}
{"item_id":"senseval2.d001.s073.t012","gold_sense_keys":["transform%2:30:00::"],"candidates":[{"index":1,"sense_key":"transform%2:30:02::","synset_id":"transform.v.01"},{"index":2,"sense_key":"transform%2:30:00::","synset_id":"transform.v.02"},{"index":3,"sense_key":"transform%2:30:03::","synset_id":"transform.v.03"},{"index":4,"sense_key":"transform%2:30:01::","synset_id":"translate.v.02"},{"index":5,"sense_key":"transform%2:30:06::","synset_id":"transform.v.05"},{"index":6,"sense_key":"transform%2:30:05::","synset_id":"transform.v.06"},{"index":7,"sense_key":"transform%2:30:04::","synset_id":"transform.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"transform%2:30:00::","call_ids":["senseval2.d001.s073.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"transform%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006240000000000001,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.234198458958417}
{"item_id":"senseval2.d001.s073.t013","gold_sense_keys":["normal%3:00:01::"],"candidates":[{"index":1,"sense_key":"normal%3:00:01::","synset_id":"normal.a.01"},{"index":2,"sense_key":"normal%3:00:02::","synset_id":"normal.a.02"},{"index":3,"sense_key":"normal%3:00:03::","synset_id":"normal.a.03"},{"index":4,"sense_key":"normal%5:00:00:perpendicular:00","synset_id":"normal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"normal%3:00:01::","call_ids":["senseval2.d001.s073.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"normal%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020789999999999998,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3534400000353344}
{"item_id":"senseval2.d001.s073.t014","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s073.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002151,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8856638749712147}
{"item_id":"senseval2.d001.s073.t015","gold_sense_keys":["cancerous%5:00:00:malignant:02"],"candidates":[{"index":1,"sense_key":"cancerous%5:00:00:malignant:02","synset_id":"cancerous.s.01"},{"index":2,"sense_key":"cancerous%5:00:00:malign:00","synset_id":"cancerous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancerous%5:00:00:malignant:02","call_ids":["senseval2.d001.s073.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancerous%5:00:00:malignant:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018004999999999999,"input_uncached_usd":0.00002005,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.953028333024122}
{"item_id":"senseval2.d001.s073.t016","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d001.s073.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006155500000000001,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.324027833004948}
{"item_id":"senseval2.d001.s074.t000","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s074.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1101,"cached_input_tokens":0,"output_tokens":2002,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008558499999999999,"input_uncached_usd":0.000055049999999999996,"input_cached_usd":0.0,"output_usd":0.0008008,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.804802500002552}
{"item_id":"senseval2.d001.s074.t001","gold_sense_keys":["exactly%4:02:01::"],"candidates":[{"index":1,"sense_key":"exactly%4:02:01::","synset_id":"precisely.r.01"},{"index":2,"sense_key":"exactly%4:02:04::","synset_id":"precisely.r.03"},{"index":3,"sense_key":"exactly%4:02:02::","synset_id":"precisely.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exactly%4:02:01::","call_ids":["senseval2.d001.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exactly%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020879999999999998,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1387871250044554}
{"item_id":"senseval2.d001.s074.t002","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d001.s074.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033715,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.901254791999236}
{"item_id":"senseval2.d001.s074.t003","gold_sense_keys":["spot%1:15:01::"],"candidates":[{"index":1,"sense_key":"spot%1:15:01::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"spot%1:10:01::","synset_id":"spot.n.02"},{"index":3,"sense_key":"spot%1:07:02::","synset_id":"point.n.14"},{"index":4,"sense_key":"spot%1:07:00::","synset_id":"smudge.n.02"},{"index":5,"sense_key":"spot%1:07:01::","synset_id":"spot.n.05"},{"index":6,"sense_key":"spot%1:10:00::","synset_id":"spot.n.06"},{"index":7,"sense_key":"spot%1:06:00::","synset_id":"spot.n.07"},{"index":8,"sense_key":"spot%1:04:01::","synset_id":"position.n.06"},{"index":9,"sense_key":"spot%1:26:00::","synset_id":"touch.n.08"},{"index":10,"sense_key":"spot%1:23:00::","synset_id":"spot.n.10"},{"index":11,"sense_key":"spot%1:10:02::","synset_id":"spot.n.11"},{"index":12,"sense_key":"spot%1:06:01::","synset_id":"spotlight.n.02"},{"index":13,"sense_key":"spot%1:06:02::","synset_id":"spot.n.13"},{"index":14,"sense_key":"spot%1:04:00::","synset_id":"blot.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spot%1:15:01::","call_ids":["senseval2.d001.s074.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spot%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":898,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002561,"input_uncached_usd":0.0000449,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.214175000030082}
{"item_id":"senseval2.d001.s075.t000","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s075.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048419999999999995,"input_uncached_usd":0.0000426,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.489616542006843}
{"item_id":"senseval2.d001.s075.t002","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["senseval2.d001.s075.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006448000000000001,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.811134625051636}
{"item_id":"senseval2.d001.s075.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s075.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029089999999999997,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3800643750000745}
{"item_id":"senseval2.d001.s075.t004","gold_sense_keys":["promote%2:41:01::"],"candidates":[{"index":1,"sense_key":"promote%2:41:01::","synset_id":"promote.v.01"},{"index":2,"sense_key":"promote%2:41:00::","synset_id":"promote.v.02"},{"index":3,"sense_key":"promote%2:32:00::","synset_id":"advertise.v.02"},{"index":4,"sense_key":"promote%2:33:01::","synset_id":"promote.v.04"},{"index":5,"sense_key":"promote%2:33:00::","synset_id":"promote.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"promote%2:41:01::","call_ids":["senseval2.d001.s075.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"promote%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.000524,"input_uncached_usd":0.0000312,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.051399582996964}
{"item_id":"senseval2.d001.s075.t005","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"growth%1:26:00::","call_ids":["senseval2.d001.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"growth%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032105,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.630443457979709}
{"item_id":"senseval2.d001.s075.t007","gold_sense_keys":["look_for%2:35:00::"],"candidates":[{"index":1,"sense_key":"look_for%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"look_for%2:37:00::","synset_id":"anticipate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look_for%2:35:00::","call_ids":["senseval2.d001.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look_for%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002854,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.44623916700948}
{"item_id":"senseval2.d001.s075.t009","gold_sense_keys":["suppress%2:30:12::"],"candidates":[{"index":1,"sense_key":"suppress%2:30:00::","synset_id":"suppress.v.01"},{"index":2,"sense_key":"suppress%2:41:01::","synset_id":"oppress.v.01"},{"index":3,"sense_key":"suppress%2:41:00::","synset_id":"inhibit.v.04"},{"index":4,"sense_key":"suppress%2:31:00::","synset_id":"suppress.v.04"},{"index":5,"sense_key":"suppress%2:30:12::","synset_id":"suppress.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"suppress%2:30:12::","call_ids":["senseval2.d001.s075.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"suppress%2:30:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005214000000000001,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.498679874988738}
{"item_id":"senseval2.d001.s075.t010","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"growth%1:26:00::","call_ids":["senseval2.d001.s075.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"growth%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032145000000000003,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.856761000002734}
{"item_id":"senseval2.d001.s076.t001","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["senseval2.d001.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034074999999999996,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.104577582969796}
{"item_id":"senseval2.d001.s076.t003","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1165,"cached_input_tokens":0,"output_tokens":2834,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.00119185,"input_uncached_usd":0.00005825,"input_cached_usd":0.0,"output_usd":0.0011336,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.049689334002323}
{"item_id":"senseval2.d001.s076.t004","gold_sense_keys":["human%3:01:01::"],"candidates":[{"index":1,"sense_key":"human%3:01:00::","synset_id":"human.a.01"},{"index":2,"sense_key":"human%3:01:01::","synset_id":"human.a.02"},{"index":3,"sense_key":"human%3:00:00::","synset_id":"human.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"human%3:01:01::","call_ids":["senseval2.d001.s076.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"human%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054225,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.450864124984946}
{"item_id":"senseval2.d001.s076.t008","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval2.d001.s076.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1165,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00098625,"input_uncached_usd":0.00005825,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.361776542034931}
{"item_id":"senseval2.d001.s076.t009","gold_sense_keys":["identical%5:00:00:same:00"],"candidates":[{"index":1,"sense_key":"identical%5:00:00:same:00","synset_id":"identical.s.01"},{"index":2,"sense_key":"identical%5:00:00:same:02","synset_id":"identical.s.02"},{"index":3,"sense_key":"identical%3:00:00::","synset_id":"identical.a.03"},{"index":4,"sense_key":"identical%5:00:00:isotropic:00","synset_id":"identical.s.03"},{"index":5,"sense_key":"identical%5:00:00:congruent:00","synset_id":"identical.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"identical%5:00:00:same:02","call_ids":["senseval2.d001.s076.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"identical%5:00:00:same:02","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044645,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.765615749987774}
{"item_id":"senseval2.d001.s076.t010","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":3,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":4,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":5,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":6,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":7,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":8,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":9,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":10,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":11,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":12,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn_out%2:42:01::","call_ids":["senseval2.d001.s076.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn_out%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035834999999999996,"input_uncached_usd":0.00004475,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.375769500038587}
{"item_id":"senseval2.d001.s076.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s076.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018869999999999998,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.73537245800253}
{"item_id":"senseval2.d001.s076.t012","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"study%1:09:02::","call_ids":["senseval2.d001.s076.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"study%1:09:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":1873,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007885499999999998,"input_uncached_usd":0.00003935,"input_cached_usd":0.0,"output_usd":0.0007492,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.39615370798856}
{"item_id":"senseval2.d001.s076.t014","gold_sense_keys":["observe%2:39:00::"],"candidates":[{"index":1,"sense_key":"observe%2:39:02::","synset_id":"detect.v.01"},{"index":2,"sense_key":"observe%2:32:00::","synset_id":"note.v.01"},{"index":3,"sense_key":"observe%2:39:01::","synset_id":"note.v.03"},{"index":4,"sense_key":"observe%2:39:00::","synset_id":"observe.v.04"},{"index":5,"sense_key":"observe%2:41:01::","synset_id":"respect.v.02"},{"index":6,"sense_key":"observe%2:41:02::","synset_id":"observe.v.06"},{"index":7,"sense_key":"observe%2:41:00::","synset_id":"watch.v.02"},{"index":8,"sense_key":"observe%2:31:00::","synset_id":"observe.v.08"},{"index":9,"sense_key":"observe%2:41:04::","synset_id":"observe.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"observe%2:39:02::","call_ids":["senseval2.d001.s076.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"observe%2:39:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00073645,"input_uncached_usd":0.000038849999999999996,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.244932917004917}
{"item_id":"senseval2.d001.s076.t015","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s076.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005414500000000001,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.837569166964386}
{"item_id":"senseval2.d001.s076.t016","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"form%1:07:03::","call_ids":["senseval2.d001.s076.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"form%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1106,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006761,"input_uncached_usd":0.000055299999999999996,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.107680000015534}
{"item_id":"senseval2.d001.s077.t000","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discovery%1:09:00::","call_ids":["senseval2.d001.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discovery%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036474999999999995,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.691383583995048}
{"item_id":"senseval2.d001.s077.t001","gold_sense_keys":["suddenly%4:02:00::","suddenly%4:02:02::"],"candidates":[{"index":1,"sense_key":"suddenly%4:02:02::","synset_id":"suddenly.r.01"},{"index":2,"sense_key":"suddenly%4:02:00::","synset_id":"abruptly.r.01"},{"index":3,"sense_key":"suddenly%4:02:04::","synset_id":"on_the_spur_of_the_moment.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suddenly%4:02:04::","call_ids":["senseval2.d001.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suddenly%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051875,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.169246416015085}
{"item_id":"senseval2.d001.s077.t002","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval2.d001.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045374999999999994,"input_uncached_usd":0.000037749999999999996,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.647347291000187}
{"item_id":"senseval2.d001.s077.t005","gold_sense_keys":["right%4:02:04::","right%4:02:09::"],"candidates":[{"index":1,"sense_key":"right%4:02:04::","synset_id":"right.r.01"},{"index":2,"sense_key":"right%4:02:08::","synset_id":"right.r.02"},{"index":3,"sense_key":"right%4:02:09::","synset_id":"right.r.03"},{"index":4,"sense_key":"right%4:02:03::","synset_id":"right.r.04"},{"index":5,"sense_key":"right%4:02:01::","synset_id":"properly.r.01"},{"index":6,"sense_key":"right%4:02:00::","synset_id":"right.r.06"},{"index":7,"sense_key":"right%4:02:07::","synset_id":"right.r.07"},{"index":8,"sense_key":"right%4:02:06::","synset_id":"mighty.r.01"},{"index":9,"sense_key":"right%4:02:05::","synset_id":"justly.r.02"},{"index":10,"sense_key":"right%4:02:02::","synset_id":"correctly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"right%4:02:04::","call_ids":["senseval2.d001.s077.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"right%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006330000000000001,"input_uncached_usd":0.0000378,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.88884195801802}
{"item_id":"senseval2.d001.s077.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s077.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011229999999999999,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9310131659731269}
{"item_id":"senseval2.d001.s077.t007","gold_sense_keys":["formation%1:22:00::"],"candidates":[{"index":1,"sense_key":"formation%1:14:00::","synset_id":"formation.n.01"},{"index":2,"sense_key":"formation%1:04:00::","synset_id":"formation.n.02"},{"index":3,"sense_key":"formation%1:04:01::","synset_id":"constitution.n.02"},{"index":4,"sense_key":"formation%1:17:00::","synset_id":"geological_formation.n.01"},{"index":5,"sense_key":"formation%1:06:00::","synset_id":"formation.n.05"},{"index":6,"sense_key":"formation%1:22:00::","synset_id":"formation.n.06"},{"index":7,"sense_key":"formation%1:04:02::","synset_id":"formation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"formation%1:22:00::","call_ids":["senseval2.d001.s077.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"formation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005497,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.688183666963596}
{"item_id":"senseval2.d001.s077.t008","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s077.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004335,"input_uncached_usd":0.0000431,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.963537665957119}
{"item_id":"senseval2.d001.s077.t009","gold_sense_keys":["leader%1:18:00::"],"candidates":[{"index":1,"sense_key":"leader%1:18:00::","synset_id":"leader.n.01"},{"index":2,"sense_key":"leader%1:06:00::","synset_id":"drawing_card.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leader%1:18:00::","call_ids":["senseval2.d001.s077.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leader%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043819999999999997,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.033881790994201}
{"item_id":"senseval2.d001.s078.t000","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"evidence%1:10:01::","call_ids":["senseval2.d001.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"evidence%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001859,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.881726042018272}
{"item_id":"senseval2.d001.s078.t001","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d001.s078.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003233,"input_uncached_usd":0.0000353,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.6497345410170965}
{"item_id":"senseval2.d001.s078.t002","gold_sense_keys":["emerge%2:30:01::"],"candidates":[{"index":1,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":2,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":3,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":4,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":5,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emerge%2:30:01::","call_ids":["senseval2.d001.s078.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emerge%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016399999999999997,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7580044170026667}
{"item_id":"senseval2.d001.s078.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s078.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016619999999999997,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6850599159952253}
{"item_id":"senseval2.d001.s078.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s078.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039335,"input_uncached_usd":0.00002855,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.074983250000514}
{"item_id":"senseval2.d001.s078.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s078.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016424999999999998,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.261674540990498}
{"item_id":"senseval2.d001.s078.t007","gold_sense_keys":["too%4:02:01::"],"candidates":[{"index":1,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"},{"index":2,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"too%4:02:01::","call_ids":["senseval2.d001.s078.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"too%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00072145,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.629777499998454}
{"item_id":"senseval2.d001.s079.t001","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"find%2:31:09::","call_ids":["senseval2.d001.s079.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"find%2:31:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006799,"input_uncached_usd":0.0000591,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.855943417001981}
{"item_id":"senseval2.d001.s079.t002","gold_sense_keys":["breast%1:08:00::"],"candidates":[{"index":1,"sense_key":"breast%1:08:01::","synset_id":"breast.n.01"},{"index":2,"sense_key":"breast%1:08:00::","synset_id":"breast.n.02"},{"index":3,"sense_key":"breast%1:13:00::","synset_id":"breast.n.03"},{"index":4,"sense_key":"breast%1:05:00::","synset_id":"breast.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"breast%1:08:00::","call_ids":["senseval2.d001.s079.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"breast%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005203,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.005034041008912}
{"item_id":"senseval2.d001.s079.t004","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s079.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004174,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.385212666995358}
{"item_id":"senseval2.d001.s079.t007","gold_sense_keys":["spot%1:15:01::"],"candidates":[{"index":1,"sense_key":"spot%1:15:01::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"spot%1:10:01::","synset_id":"spot.n.02"},{"index":3,"sense_key":"spot%1:07:02::","synset_id":"point.n.14"},{"index":4,"sense_key":"spot%1:07:00::","synset_id":"smudge.n.02"},{"index":5,"sense_key":"spot%1:07:01::","synset_id":"spot.n.05"},{"index":6,"sense_key":"spot%1:10:00::","synset_id":"spot.n.06"},{"index":7,"sense_key":"spot%1:06:00::","synset_id":"spot.n.07"},{"index":8,"sense_key":"spot%1:04:01::","synset_id":"position.n.06"},{"index":9,"sense_key":"spot%1:26:00::","synset_id":"touch.n.08"},{"index":10,"sense_key":"spot%1:23:00::","synset_id":"spot.n.10"},{"index":11,"sense_key":"spot%1:10:02::","synset_id":"spot.n.11"},{"index":12,"sense_key":"spot%1:06:01::","synset_id":"spotlight.n.02"},{"index":13,"sense_key":"spot%1:06:02::","synset_id":"spot.n.13"},{"index":14,"sense_key":"spot%1:04:00::","synset_id":"blot.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spot%1:15:01::","call_ids":["senseval2.d001.s079.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spot%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038815,"input_uncached_usd":0.00004895,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.88773316697916}
{"item_id":"senseval2.d001.s080.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006898,"input_uncached_usd":0.0000434,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.728309666039422}
{"item_id":"senseval2.d001.s080.t004","gold_sense_keys":["often%4:02:00::","often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d001.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036399999999999996,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.878562666010112}
{"item_id":"senseval2.d001.s080.t005","gold_sense_keys":["strike%2:35:02::"],"candidates":[{"index":1,"sense_key":"strike%2:35:00::","synset_id":"strike.v.01"},{"index":2,"sense_key":"strike%2:37:00::","synset_id":"affect.v.05"},{"index":3,"sense_key":"strike%2:35:01::","synset_id":"hit.v.02"},{"index":4,"sense_key":"strike%2:33:00::","synset_id":"strike.v.04"},{"index":5,"sense_key":"strike%2:32:00::","synset_id":"strike.v.05"},{"index":6,"sense_key":"strike%2:35:02::","synset_id":"hit.v.05"},{"index":7,"sense_key":"strike%2:41:00::","synset_id":"strike.v.07"},{"index":8,"sense_key":"strike%2:42:00::","synset_id":"fall.v.08"},{"index":9,"sense_key":"strike%2:41:02::","synset_id":"come_to.v.03"},{"index":10,"sense_key":"strike%2:35:03::","synset_id":"strike.v.10"},{"index":11,"sense_key":"strike%2:30:00::","synset_id":"strike.v.11"},{"index":12,"sense_key":"strike%2:40:00::","synset_id":"fall_upon.v.01"},{"index":13,"sense_key":"strike%2:36:00::","synset_id":"strike.v.13"},{"index":14,"sense_key":"strike%2:35:08::","synset_id":"strike.v.14"},{"index":15,"sense_key":"strike%2:39:00::","synset_id":"hit.v.09"},{"index":16,"sense_key":"strike%2:38:02::","synset_id":"hit.v.12"},{"index":17,"sense_key":"strike%2:38:08::","synset_id":"assume.v.05"},{"index":18,"sense_key":"strike%2:36:02::","synset_id":"mint.v.01"},{"index":19,"sense_key":"strike%2:35:10::","synset_id":"strickle.v.02"},{"index":20,"sense_key":"strike%2:35:09::","synset_id":"strike.v.20"},{"index":21,"sense_key":"strike%2:31:00::","synset_id":"strike.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"strike%2:35:02::","call_ids":["senseval2.d001.s080.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"strike%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1349,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0006118499999999999,"input_uncached_usd":0.00006745,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.763710500032175}
{"item_id":"senseval2.d001.s080.t007","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval2.d001.s080.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001406,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4722413329873234}
{"item_id":"senseval2.d001.s080.t008","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["senseval2.d001.s080.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032335,"input_uncached_usd":0.00003535,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1068410410080105}
{"item_id":"senseval2.d001.s080.t009","gold_sense_keys":["family%1:14:01::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"family%1:14:01::","call_ids":["senseval2.d001.s080.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"family%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005048499999999999,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.829931167012546}
{"item_id":"senseval2.d001.s080.t011","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s080.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010754999999999999,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2395063749863766}
{"item_id":"senseval2.d001.s080.t012","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s080.t012__v1__a1","senseval2.d001.s080.t012__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00042084999999999996,"input_uncached_usd":0.00004965,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.911497458058875}
{"item_id":"senseval2.d001.s080.t013","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"form%1:14:02::","call_ids":["senseval2.d001.s080.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"form%1:14:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1117,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00044625,"input_uncached_usd":0.000055849999999999996,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0732363749993965}
{"item_id":"senseval2.d001.s080.t014","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:14:00::","call_ids":["senseval2.d001.s080.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029154999999999996,"input_uncached_usd":0.000029149999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0473164999857545}
{"item_id":"senseval2.d001.s080.t015","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s080.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013805,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.047043833998032}
{"item_id":"senseval2.d001.s081.t002","gold_sense_keys":["implicate%2:42:01::"],"candidates":[{"index":1,"sense_key":"implicate%2:42:01::","synset_id":"implicate.v.01"},{"index":2,"sense_key":"implicate%2:42:00::","synset_id":"entail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"implicate%2:42:01::","call_ids":["senseval2.d001.s081.t002__v1__a1","senseval2.d001.s081.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"implicate%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":999,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00065155,"input_uncached_usd":0.000049949999999999994,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.899768833012786}
{"item_id":"senseval2.d001.s082.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["senseval2.d001.s082.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001682,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.07698412501486}
{"item_id":"senseval2.d001.s082.t001","gold_sense_keys":["last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%5:00:00:past:00","call_ids":["senseval2.d001.s082.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005825999999999998,"input_uncached_usd":0.0000382,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.850251625000965}
{"item_id":"senseval2.d001.s082.t002","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["senseval2.d001.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033654999999999997,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3136715409928}
{"item_id":"senseval2.d001.s082.t004","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval2.d001.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":2576,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010720999999999999,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.0010304,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.668399665970355}
{"item_id":"senseval2.d001.s082.t005","gold_sense_keys":["about%4:02:00::"],"candidates":[{"index":1,"sense_key":"about%4:02:00::","synset_id":"approximately.r.01"},{"index":2,"sense_key":"about%4:02:06::","synset_id":"about.r.02"},{"index":3,"sense_key":"about%4:02:01::","synset_id":"about.r.03"},{"index":4,"sense_key":"about%4:02:04::","synset_id":"about.r.04"},{"index":5,"sense_key":"about%4:02:02::","synset_id":"about.r.05"},{"index":6,"sense_key":"about%4:02:03::","synset_id":"about.r.06"},{"index":7,"sense_key":"about%4:02:05::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"about%4:02:00::","call_ids":["senseval2.d001.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"about%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007313,"input_uncached_usd":0.0000337,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.08228512504138}
{"item_id":"senseval2.d001.s082.t006","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["senseval2.d001.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037039999999999995,"input_uncached_usd":0.0000312,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.436233834014274}
{"item_id":"senseval2.d001.s082.t007","gold_sense_keys":["take%2:30:00::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"take%2:30:00::","call_ids":["senseval2.d001.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"take%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2214,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0006291000000000001,"input_uncached_usd":0.0001107,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1473905830061994}
{"item_id":"senseval2.d001.s082.t010","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"},{"index":2,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015509999999999998,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5395558750024065}
{"item_id":"senseval2.d001.s082.t011","gold_sense_keys":["test%2:32:08::"],"candidates":[{"index":1,"sense_key":"test%2:41:00::","synset_id":"test.v.01"},{"index":2,"sense_key":"test%2:41:01::","synset_id":"screen.v.01"},{"index":3,"sense_key":"test%2:32:00::","synset_id":"quiz.v.01"},{"index":4,"sense_key":"test%2:42:00::","synset_id":"test.v.04"},{"index":5,"sense_key":"test%2:33:00::","synset_id":"test.v.05"},{"index":6,"sense_key":"test%2:32:08::","synset_id":"test.v.06"},{"index":7,"sense_key":"test%2:31:06::","synset_id":"test.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"test%2:32:08::","call_ids":["senseval2.d001.s082.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"test%2:32:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":2960,"reasoning_output_tokens":2944},"cost":{"currency":"USD","total_usd":0.0012151,"input_uncached_usd":0.0000311,"input_cached_usd":0.0,"output_usd":0.001184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.99691358394921}
{"item_id":"senseval2.d001.s082.t012","gold_sense_keys":["miss%2:42:00::"],"candidates":[{"index":1,"sense_key":"miss%2:39:00::","synset_id":"miss.v.01"},{"index":2,"sense_key":"miss%2:37:00::","synset_id":"miss.v.02"},{"index":3,"sense_key":"miss%2:42:01::","synset_id":"miss.v.03"},{"index":4,"sense_key":"miss%2:31:00::","synset_id":"neglect.v.01"},{"index":5,"sense_key":"miss%2:38:00::","synset_id":"miss.v.05"},{"index":6,"sense_key":"miss%2:42:00::","synset_id":"miss.v.06"},{"index":7,"sense_key":"miss%2:35:00::","synset_id":"miss.v.07"},{"index":8,"sense_key":"miss%2:42:02::","synset_id":"miss.v.08"},{"index":9,"sense_key":"miss%2:32:00::","synset_id":"miss.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"miss%2:42:02::","call_ids":["senseval2.d001.s082.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"miss%2:42:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039914999999999997,"input_uncached_usd":0.00003435,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.393672667036299}
{"item_id":"senseval2.d001.s083.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["senseval2.d001.s083.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044885,"input_uncached_usd":0.00003285,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.432916874997318}
{"item_id":"senseval2.d001.s083.t003","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":2,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":3,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":4,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"},{"index":5,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":6,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"yet%4:02:04::","call_ids":["senseval2.d001.s083.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"yet%4:02:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007823999999999999,"input_uncached_usd":0.0000336,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.138455000007525}
{"item_id":"senseval2.d001.s083.t007","gold_sense_keys":["tissue%1:08:00::"],"candidates":[{"index":1,"sense_key":"tissue%1:08:00::","synset_id":"tissue.n.01"},{"index":2,"sense_key":"tissue%1:27:02::","synset_id":"tissue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tissue%1:08:00::","call_ids":["senseval2.d001.s083.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tissue%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017974999999999998,"input_uncached_usd":0.00001975,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5397888750303537}
{"item_id":"senseval2.d001.s083.t008","gold_sense_keys":["take%2:30:00::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"take%2:30:00::","call_ids":["senseval2.d001.s083.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"take%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2195,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0007565499999999999,"input_uncached_usd":0.00010975,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.39744166698074}
{"item_id":"senseval2.d001.s083.t010","gold_sense_keys":["brain%1:08:00::"],"candidates":[{"index":1,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":2,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":5,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brain%1:08:00::","call_ids":["senseval2.d001.s083.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brain%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034185,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.389487291045953}
{"item_id":"senseval2.d001.s084.t000","gold_sense_keys":["at_the_same_time%4:02:05::"],"candidates":[{"index":1,"sense_key":"at_the_same_time%4:02:03::","synset_id":"simultaneously.r.01"},{"index":2,"sense_key":"at_the_same_time%4:02:05::","synset_id":"concurrently.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"at_the_same_time%4:02:03::","call_ids":["senseval2.d001.s084.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"at_the_same_time%4:02:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005656,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.713287207996473}
{"item_id":"senseval2.d001.s084.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["senseval2.d001.s084.t001__v1__a1","senseval2.d001.s084.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":609,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00028485,"input_uncached_usd":0.00004125,"input_cached_usd":0.0,"output_usd":0.00024359999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.660764333035331}
{"item_id":"senseval2.d001.s084.t002","gold_sense_keys":["rush%2:30:00::"],"candidates":[{"index":1,"sense_key":"rush%2:38:00::","synset_id":"rush.v.01"},{"index":2,"sense_key":"rush%2:33:00::","synset_id":"rush.v.02"},{"index":3,"sense_key":"rush%2:30:01::","synset_id":"rush.v.03"},{"index":4,"sense_key":"rush%2:30:00::","synset_id":"rush.v.04"},{"index":5,"sense_key":"rush%2:38:01::","synset_id":"rush.v.05"},{"index":6,"sense_key":"rush%2:38:10::","synset_id":"race.v.04"},{"index":7,"sense_key":"rush%2:36:00::","synset_id":"induce.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rush%2:30:01::","call_ids":["senseval2.d001.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rush%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039485,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.226225749996956}
{"item_id":"senseval2.d001.s084.t003","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s084.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018534999999999998,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0965956659638323}
{"item_id":"senseval2.d001.s085.t000","gold_sense_keys":["hope%2:31:00::"],"candidates":[{"index":1,"sense_key":"hope%2:37:01::","synset_id":"hope.v.01"},{"index":2,"sense_key":"hope%2:37:00::","synset_id":"hope.v.02"},{"index":3,"sense_key":"hope%2:31:00::","synset_id":"hope.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hope%2:31:00::","call_ids":["senseval2.d001.s085.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hope%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026015,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.761398917005863}
{"item_id":"senseval2.d001.s085.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s085.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004191,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.726082833018154}
{"item_id":"senseval2.d001.s086.t001","gold_sense_keys":["close%3:00:01::","close%3:00:05::"],"candidates":[{"index":1,"sense_key":"close%3:00:01::","synset_id":"close.a.01"},{"index":2,"sense_key":"close%3:00:02::","synset_id":"close.a.02"},{"index":3,"sense_key":"close%3:00:05::","synset_id":"near.a.01"},{"index":4,"sense_key":"close%5:00:00:careful:00","synset_id":"close.s.01"},{"index":5,"sense_key":"close%5:00:00:accurate:00","synset_id":"close.s.02"},{"index":6,"sense_key":"close%5:00:00:equal:00","synset_id":"close.s.03"},{"index":7,"sense_key":"close%5:00:00:confined:00","synset_id":"close.s.04"},{"index":8,"sense_key":"close%5:00:00:unventilated:00","synset_id":"airless.s.01"},{"index":9,"sense_key":"close%5:00:00:fine:00","synset_id":"close.s.06"},{"index":10,"sense_key":"close%5:00:00:restrained:00","synset_id":"close.s.07"},{"index":11,"sense_key":"close%5:00:00:private:00","synset_id":"close.s.08"},{"index":12,"sense_key":"close%5:00:00:tight:01","synset_id":"close.s.09"},{"index":13,"sense_key":"close%5:00:00:short:01","synset_id":"close.s.10"},{"index":14,"sense_key":"close%5:00:00:stingy:00","synset_id":"cheeseparing.s.01"},{"index":15,"sense_key":"close%5:00:00:uncommunicative:00","synset_id":"close.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"close%3:00:05::","call_ids":["senseval2.d001.s086.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"close%3:00:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":963,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00023375,"input_uncached_usd":0.00004815,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.772170541982632}
{"item_id":"senseval2.d001.s086.t002","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:36:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"find%2:36:00::","call_ids":["senseval2.d001.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"find%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1123,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00072815,"input_uncached_usd":0.000056149999999999996,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.419744791986886}
{"item_id":"senseval2.d001.s086.t004","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034209999999999997,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.678249666991178}
{"item_id":"senseval2.d001.s086.t005","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["senseval2.d001.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018839999999999997,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.380145250004716}
{"item_id":"senseval2.d001.s087.t000","gold_sense_keys":["believe%2:31:04::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"believe%2:31:03::","call_ids":["senseval2.d001.s087.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"believe%2:31:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002659,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4414315419853665}
{"item_id":"senseval2.d001.s087.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d001.s087.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039139999999999997,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.150241416005883}
{"item_id":"senseval2.d001.s087.t002","gold_sense_keys":["inherit%2:40:01::"],"candidates":[{"index":1,"sense_key":"inherit%2:40:00::","synset_id":"inherit.v.01"},{"index":2,"sense_key":"inherit%2:40:02::","synset_id":"inherit.v.02"},{"index":3,"sense_key":"inherit%2:40:01::","synset_id":"inherit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"inherit%2:40:01::","call_ids":["senseval2.d001.s087.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"inherit%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00008035,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.6948183749918826}
{"item_id":"senseval2.d001.s087.t006","gold_sense_keys":["copy%1:06:00::"],"candidates":[{"index":1,"sense_key":"copy%1:10:00::","synset_id":"transcript.n.02"},{"index":2,"sense_key":"copy%1:06:00::","synset_id":"copy.n.02"},{"index":3,"sense_key":"copy%1:10:01::","synset_id":"copy.n.03"},{"index":4,"sense_key":"copy%1:10:02::","synset_id":"copy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"copy%1:06:00::","call_ids":["senseval2.d001.s087.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"copy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023639999999999997,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.777310292003676}
{"item_id":"senseval2.d001.s087.t008","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d001.s087.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023304999999999997,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.508537166984752}
{"item_id":"senseval2.d001.s088.t001","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["senseval2.d001.s088.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024045,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.265993457986042}
{"item_id":"senseval2.d001.s088.t005","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d001.s088.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034205,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.014709540992044}
{"item_id":"senseval2.d001.s088.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d001.s088.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023639999999999997,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3778370830114}
{"item_id":"senseval2.d001.s088.t009","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d001.s088.t009__v1__a1","senseval2.d001.s088.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":2272,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009526500000000001,"input_uncached_usd":0.000043849999999999995,"input_cached_usd":0.0,"output_usd":0.0009088,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.670681709016208}
{"item_id":"senseval2.d001.s088.t010","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"common%3:00:01::","call_ids":["senseval2.d001.s088.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"common%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":1618,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006857499999999999,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.0006472,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.818871749972459}
{"item_id":"senseval2.d001.s088.t011","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s088.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013529999999999998,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.149761207983829}
{"item_id":"senseval2.d001.s089.t000","gold_sense_keys":["discovery%1:04:00::","discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s089.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033759999999999997,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.743158041965216}
{"item_id":"senseval2.d001.s089.t003","gold_sense_keys":["only%4:02:02::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"only%4:02:02::","call_ids":["senseval2.d001.s089.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"only%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002431,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7637493340298533}
{"item_id":"senseval2.d001.s089.t004","gold_sense_keys":["speculate%2:31:01::","speculate%2:32:00::"],"candidates":[{"index":1,"sense_key":"speculate%2:31:01::","synset_id":"speculate.v.01"},{"index":2,"sense_key":"speculate%2:32:00::","synset_id":"speculate.v.02"},{"index":3,"sense_key":"speculate%2:31:00::","synset_id":"chew_over.v.01"},{"index":4,"sense_key":"speculate%2:40:00::","synset_id":"speculate.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"speculate%2:31:01::","call_ids":["senseval2.d001.s089.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"speculate%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039464999999999997,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.204675374960061}
{"item_id":"senseval2.d001.s090.t001","gold_sense_keys":["major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval2.d001.s090.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049805,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.908177125034854}
{"item_id":"senseval2.d001.s090.t004","gold_sense_keys":["unit%1:14:00::"],"candidates":[{"index":1,"sense_key":"unit%1:23:00::","synset_id":"unit_of_measurement.n.01"},{"index":2,"sense_key":"unit%1:24:00::","synset_id":"unit.n.02"},{"index":3,"sense_key":"unit%1:14:00::","synset_id":"unit.n.03"},{"index":4,"sense_key":"unit%1:09:00::","synset_id":"unit.n.04"},{"index":5,"sense_key":"unit%1:17:00::","synset_id":"unit.n.05"},{"index":6,"sense_key":"unit%1:03:00::","synset_id":"whole.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unit%1:24:00::","call_ids":["senseval2.d001.s090.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unit%1:24:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":2448,"reasoning_output_tokens":2432},"cost":{"currency":"USD","total_usd":0.00101205,"input_uncached_usd":0.00003285,"input_cached_usd":0.0,"output_usd":0.0009792,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.452798207988963}
{"item_id":"senseval2.d001.s090.t005","gold_sense_keys":["collaborate%2:41:00::"],"candidates":[{"index":1,"sense_key":"collaborate%2:41:00::","synset_id":"collaborate.v.01"},{"index":2,"sense_key":"collaborate%2:41:01::","synset_id":"collaborate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"collaborate%2:41:00::","call_ids":["senseval2.d001.s090.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"collaborate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048985,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.112353416974656}
{"item_id":"senseval2.d001.s090.t007","gold_sense_keys":["hunter%1:18:01::"],"candidates":[{"index":1,"sense_key":"hunter%1:18:00::","synset_id":"hunter.n.01"},{"index":2,"sense_key":"hunter%1:18:01::","synset_id":"hunter.n.02"},{"index":3,"sense_key":"hunter%1:17:00::","synset_id":"orion.n.02"},{"index":4,"sense_key":"hunter%1:06:00::","synset_id":"hunter.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hunter%1:18:01::","call_ids":["senseval2.d001.s090.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hunter%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015849999999999998,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8416933339904062}
{"item_id":"senseval2.d001.s090.t008","gold_sense_keys":["turn%2:42:13::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"turn%2:42:13::","call_ids":["senseval2.d001.s090.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"turn%2:42:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1478,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00020829999999999997,"input_uncached_usd":0.0000739,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.662540125020314}
{"item_id":"senseval2.d001.s090.t010","gold_sense_keys":["discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["senseval2.d001.s090.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038899999999999997,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.597242833988275}
{"item_id":"senseval2.d001.s090.t014","gold_sense_keys":["new%3:00:00::","new%5:00:00:original:00","new%5:00:00:other:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s090.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005347,"input_uncached_usd":0.000041899999999999995,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.642517041997053}
{"item_id":"senseval2.d001.s091.t001","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"say%2:32:13::","call_ids":["senseval2.d001.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"say%2:32:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066275,"input_uncached_usd":0.000041949999999999996,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.410852750006597}
{"item_id":"senseval2.d001.s091.t002","gold_sense_keys":["new%3:00:00::","new%3:00:09::","new%5:00:00:original:00","new%5:00:00:unprecedented:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d001.s091.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006372000000000001,"input_uncached_usd":0.000042,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.243613290949725}
{"item_id":"senseval2.d001.s091.t005","gold_sense_keys":["slow%2:30:05::"],"candidates":[{"index":1,"sense_key":"slow%2:30:00::","synset_id":"decelerate.v.01"},{"index":2,"sense_key":"slow%2:30:01::","synset_id":"slow.v.02"},{"index":3,"sense_key":"slow%2:30:05::","synset_id":"slow.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"slow%2:30:05::","call_ids":["senseval2.d001.s091.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"slow%2:30:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018395,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.934149292006623}
{"item_id":"senseval2.d001.s091.t006","gold_sense_keys":["reverse%2:30:00::"],"candidates":[{"index":1,"sense_key":"reverse%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":2,"sense_key":"reverse%2:30:01::","synset_id":"turn_back.v.05"},{"index":3,"sense_key":"reverse%2:31:00::","synset_id":"overrule.v.01"},{"index":4,"sense_key":"reverse%2:32:00::","synset_id":"revoke.v.02"},{"index":5,"sense_key":"reverse%2:30:03::","synset_id":"invert.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reverse%2:30:00::","call_ids":["senseval2.d001.s091.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reverse%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036754999999999996,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.120853542000987}
{"item_id":"senseval2.d001.s091.t008","gold_sense_keys":["growth%1:22:00::","growth%1:26:00::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"growth%1:26:00::","call_ids":["senseval2.d001.s091.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"growth%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047404999999999995,"input_uncached_usd":0.000032449999999999996,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.880355375004001}
{"item_id":"senseval2.d001.s091.t009","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["senseval2.d001.s091.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018355,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.911762874980923}
{"item_id":"senseval2.d001.s091.t012","gold_sense_keys":["produce%2:36:02::","produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"produce%2:36:02::","call_ids":["senseval2.d001.s091.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"produce%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027164999999999996,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4962966670282185}
{"item_id":"senseval2.d001.s092.t000","gold_sense_keys":["idea%1:09:01::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"idea%1:09:01::","call_ids":["senseval2.d001.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"idea%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003174,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.15083808300551}
{"item_id":"senseval2.d001.s092.t001","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"},{"index":2,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":3,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":4,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":5,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"administer%2:29:00::","call_ids":["senseval2.d001.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001624,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1683355419663712}
{"item_id":"senseval2.d001.s092.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["senseval2.d001.s092.t002__v1__a1","senseval2.d001.s092.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":909,"cached_input_tokens":0,"output_tokens":2080,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008774499999999999,"input_uncached_usd":0.000045449999999999993,"input_cached_usd":0.0,"output_usd":0.000832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.020618041977286}
{"item_id":"senseval2.d001.s092.t005","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s092.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026605,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.364380666986108}
{"item_id":"senseval2.d001.s092.t006","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s092.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036884999999999994,"input_uncached_usd":0.00002965,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.687768167001195}
{"item_id":"senseval2.d001.s092.t007","gold_sense_keys":["damaged%3:00:00::"],"candidates":[{"index":1,"sense_key":"damaged%3:00:00::","synset_id":"damaged.a.01"},{"index":2,"sense_key":"damaged%5:00:00:disreputable:00","synset_id":"discredited.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damaged%3:00:00::","call_ids":["senseval2.d001.s092.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damaged%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.000412,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.151944666984491}
{"item_id":"senseval2.d001.s093.t000","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["senseval2.d001.s093.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046254999999999994,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.757786540954839}
{"item_id":"senseval2.d001.s093.t001","gold_sense_keys":["replace%2:30:00::"],"candidates":[{"index":1,"sense_key":"replace%2:30:00::","synset_id":"replace.v.01"},{"index":2,"sense_key":"replace%2:41:00::","synset_id":"supplant.v.01"},{"index":3,"sense_key":"replace%2:35:00::","synset_id":"replace.v.03"},{"index":4,"sense_key":"replace%2:40:00::","synset_id":"substitute.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"replace%2:40:00::","call_ids":["senseval2.d001.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"replace%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007781999999999999,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.094704542018007}
{"item_id":"senseval2.d001.s093.t004","gold_sense_keys":["healthy%3:00:00::"],"candidates":[{"index":1,"sense_key":"healthy%3:00:00::","synset_id":"healthy.a.01"},{"index":2,"sense_key":"healthy%5:00:00:sound:00","synset_id":"healthy.s.01"},{"index":3,"sense_key":"healthy%5:00:00:wholesome:00","synset_id":"healthy.s.02"},{"index":4,"sense_key":"healthy%5:00:00:reasonable:00","synset_id":"healthy.s.03"},{"index":5,"sense_key":"healthy%5:00:00:considerable:00","synset_id":"goodly.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"healthy%3:00:00::","call_ids":["senseval2.d001.s093.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"healthy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026625,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7934999999706633}
{"item_id":"senseval2.d001.s093.t005","gold_sense_keys":["version%1:09:01::"],"candidates":[{"index":1,"sense_key":"version%1:10:01::","synset_id":"version.n.01"},{"index":2,"sense_key":"version%1:09:01::","synset_id":"version.n.02"},{"index":3,"sense_key":"version%1:10:02::","synset_id":"adaptation.n.01"},{"index":4,"sense_key":"version%1:10:00::","synset_id":"translation.n.01"},{"index":5,"sense_key":"version%1:09:00::","synset_id":"interpretation.n.01"},{"index":6,"sense_key":"version%1:04:00::","synset_id":"version.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"version%1:09:01::","call_ids":["senseval2.d001.s093.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"version%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003178,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.349269708036445}
{"item_id":"senseval2.d001.s093.t006","gold_sense_keys":["come_close%2:41:00::"],"candidates":[{"index":1,"sense_key":"come_close%2:41:00::","synset_id":"come_close.v.01"},{"index":2,"sense_key":"come_close%2:42:00::","synset_id":"approximate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"come_close%2:41:00::","call_ids":["senseval2.d001.s093.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"come_close%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064165,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.537912875006441}
{"item_id":"senseval2.d001.s094.t001","gold_sense_keys":["say%2:32:00::","say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval2.d001.s094.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035339999999999997,"input_uncached_usd":0.0000398,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.669154417002574}
{"item_id":"senseval2.d001.s094.t002","gold_sense_keys":["witness%2:39:01::"],"candidates":[{"index":1,"sense_key":"witness%2:39:00::","synset_id":"witness.v.01"},{"index":2,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"witness%2:39:01::","call_ids":["senseval2.d001.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"witness%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035904999999999997,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.72278341697529}
{"item_id":"senseval2.d001.s094.t003","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discovery%1:09:00::","call_ids":["senseval2.d001.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discovery%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010534999999999999,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0242999590118416}
{"item_id":"senseval2.d001.s094.t004","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d001.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023354999999999998,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4092697920277715}
{"item_id":"senseval2.d001.s094.t005","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"important%3:00:02::","call_ids":["senseval2.d001.s094.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"important%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003405,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.097639416984748}
{"item_id":"senseval2.d001.s094.t006","gold_sense_keys":["step%1:04:02::"],"candidates":[{"index":1,"sense_key":"step%1:04:02::","synset_id":"measure.n.01"},{"index":2,"sense_key":"step%1:23:00::","synset_id":"footstep.n.03"},{"index":3,"sense_key":"step%1:04:00::","synset_id":"step.n.03"},{"index":4,"sense_key":"step%1:06:00::","synset_id":"step.n.04"},{"index":5,"sense_key":"step%1:26:00::","synset_id":"gradation.n.01"},{"index":6,"sense_key":"step%1:23:01::","synset_id":"step.n.06"},{"index":7,"sense_key":"step%1:11:00::","synset_id":"footfall.n.01"},{"index":8,"sense_key":"step%1:10:00::","synset_id":"tone.n.09"},{"index":9,"sense_key":"step%1:10:02::","synset_id":"footprint.n.01"},{"index":10,"sense_key":"step%1:06:01::","synset_id":"step.n.10"},{"index":11,"sense_key":"step%1:04:03::","synset_id":"dance_step.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"step%1:04:00::","call_ids":["senseval2.d001.s094.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"step%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037814999999999995,"input_uncached_usd":0.00003895,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.190410459006671}
{"item_id":"senseval2.d001.s094.t007","gold_sense_keys":["genesis%1:11:00::"],"candidates":[{"index":1,"sense_key":"genesis%1:11:00::","synset_id":"genesis.n.01"},{"index":2,"sense_key":"genesis%1:10:00::","synset_id":"genesis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"genesis%1:11:00::","call_ids":["senseval2.d001.s094.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"genesis%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012934999999999997,"input_uncached_usd":0.000020549999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.897088832978625}
{"item_id":"senseval2.d001.s094.t008","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["senseval2.d001.s094.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001345,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.617561333987396}
{"item_id":"senseval2.d002.s000.t000","gold_sense_keys":["teach%2:32:00::"],"candidates":[{"index":1,"sense_key":"teach%2:32:00::","synset_id":"teach.v.01"},{"index":2,"sense_key":"teach%2:30:00::","synset_id":"teach.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teach%2:32:00::","call_ids":["senseval2.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teach%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":256,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005828,"input_uncached_usd":0.0000128,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.251482792024035}
{"item_id":"senseval2.d002.s000.t001","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007151000000000001,"input_uncached_usd":0.0000175,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.080781208991539}
{"item_id":"senseval2.d002.s000.t002","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["senseval2.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037194999999999996,"input_uncached_usd":0.000032749999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.072247083007824}
{"item_id":"senseval2.d002.s000.t003","gold_sense_keys":["write%2:36:01::"],"candidates":[{"index":1,"sense_key":"write%2:36:00::","synset_id":"write.v.01"},{"index":2,"sense_key":"write%2:32:00::","synset_id":"write.v.02"},{"index":3,"sense_key":"write%2:36:02::","synset_id":"publish.v.03"},{"index":4,"sense_key":"write%2:32:08::","synset_id":"write.v.04"},{"index":5,"sense_key":"write%2:32:01::","synset_id":"write.v.05"},{"index":6,"sense_key":"write%2:36:04::","synset_id":"compose.v.02"},{"index":7,"sense_key":"write%2:36:01::","synset_id":"write.v.07"},{"index":8,"sense_key":"write%2:32:03::","synset_id":"write.v.08"},{"index":9,"sense_key":"write%2:36:03::","synset_id":"spell.v.03"},{"index":10,"sense_key":"write%2:36:10::","synset_id":"write.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"write%2:32:00::","call_ids":["senseval2.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"write%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044494999999999995,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.04910220799502}
{"item_id":"senseval2.d002.s000.t004","gold_sense_keys":["reckon%2:31:01::"],"candidates":[{"index":1,"sense_key":"reckon%2:31:02::","synset_id":"think.v.02"},{"index":2,"sense_key":"reckon%2:31:03::","synset_id":"calculate.v.02"},{"index":3,"sense_key":"reckon%2:31:00::","synset_id":"see.v.05"},{"index":4,"sense_key":"reckon%2:31:01::","synset_id":"calculate.v.01"},{"index":5,"sense_key":"reckon%2:31:05::","synset_id":"count.v.08"},{"index":6,"sense_key":"reckon%2:31:13::","synset_id":"reckon.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"reckon%2:31:01::","call_ids":["senseval2.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"reckon%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003108,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.9831239999621175}
{"item_id":"senseval2.d002.s001.t000","gold_sense_keys":["know%2:31:03::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"know%2:31:03::","call_ids":["senseval2.d002.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"know%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024615,"input_uncached_usd":0.000034949999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.845752125023864}
{"item_id":"senseval2.d002.s002.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002326,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.709537292015739}
{"item_id":"senseval2.d002.s003.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reason%1:16:00::","call_ids":["senseval2.d002.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reason%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046869999999999995,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.051665167033207}
{"item_id":"senseval2.d002.s003.t001","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023409999999999997,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.140961208031513}
{"item_id":"senseval2.d002.s003.t002","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effective%5:00:00:efficacious:00","call_ids":["senseval2.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effective%5:00:00:efficacious:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005987000000000001,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.74696399999084}
{"item_id":"senseval2.d002.s003.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005704,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.936368208029307}
{"item_id":"senseval2.d002.s003.t004","gold_sense_keys":["require%2:42:00::"],"candidates":[{"index":1,"sense_key":"require%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"require%2:32:01::","synset_id":"ask.v.04"},{"index":3,"sense_key":"require%2:32:00::","synset_id":"command.v.02"},{"index":4,"sense_key":"require%2:34:00::","synset_id":"want.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"require%2:32:01::","call_ids":["senseval2.d002.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"require%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006933499999999999,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.136602166981902}
{"item_id":"senseval2.d002.s003.t005","gold_sense_keys":["relinquish%2:41:00::"],"candidates":[{"index":1,"sense_key":"relinquish%2:40:01::","synset_id":"release.v.08"},{"index":2,"sense_key":"relinquish%2:41:00::","synset_id":"waive.v.01"},{"index":3,"sense_key":"relinquish%2:40:00::","synset_id":"foreswear.v.02"},{"index":4,"sense_key":"relinquish%2:35:02::","synset_id":"let_go_of.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"relinquish%2:41:00::","call_ids":["senseval2.d002.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"relinquish%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023564999999999998,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.134581000020262}
{"item_id":"senseval2.d002.s003.t008","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval2.d002.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":338,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003049,"input_uncached_usd":0.0000169,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.234164417022839}
{"item_id":"senseval2.d002.s003.t013","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s003.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031215,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4746922079939395}
{"item_id":"senseval2.d002.s003.t014","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s003.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036114999999999997,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.022686125012115}
{"item_id":"senseval2.d002.s003.t016","gold_sense_keys":["violate%2:41:00::"],"candidates":[{"index":1,"sense_key":"violate%2:42:00::","synset_id":"violate.v.01"},{"index":2,"sense_key":"violate%2:41:00::","synset_id":"transgress.v.01"},{"index":3,"sense_key":"violate%2:35:00::","synset_id":"violate.v.03"},{"index":4,"sense_key":"violate%2:41:02::","synset_id":"desecrate.v.01"},{"index":5,"sense_key":"violate%2:41:01::","synset_id":"rape.v.01"},{"index":6,"sense_key":"violate%2:35:01::","synset_id":"rape.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"violate%2:41:00::","call_ids":["senseval2.d002.s003.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"violate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041855,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.01418499997817}
{"item_id":"senseval2.d002.s004.t000","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval2.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020275,"input_uncached_usd":0.00001715,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.000638625002466}
{"item_id":"senseval2.d002.s004.t001","gold_sense_keys":["so%4:02:03::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"so%4:02:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042785,"input_uncached_usd":0.000037449999999999995,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.775584791961592}
{"item_id":"senseval2.d002.s004.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":330,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004069,"input_uncached_usd":0.000016499999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.155970250023529}
{"item_id":"senseval2.d002.s004.t004","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":3,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":4,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":5,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":6,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":7,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"establishment%1:14:00::","call_ids":["senseval2.d002.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"establishment%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008021500000000001,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.393422958033625}
{"item_id":"senseval2.d002.s004.t005","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015225,"input_uncached_usd":0.00001785,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9594537089578807}
{"item_id":"senseval2.d002.s004.t006","gold_sense_keys":["even%4:02:00::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"even%4:02:00::","call_ids":["senseval2.d002.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"even%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036079999999999993,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.482808417000342}
{"item_id":"senseval2.d002.s004.t007","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030515,"input_uncached_usd":0.00001715,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.548485375009477}
{"item_id":"senseval2.d002.s004.t008","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval2.d002.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004129,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.085871124989353}
{"item_id":"senseval2.d002.s004.t010","gold_sense_keys":["blasphemous%5:00:00:irreverent:00"],"candidates":[{"index":1,"sense_key":"blasphemous%5:00:00:irreverent:00","synset_id":"blasphemous.s.01"},{"index":2,"sense_key":"blasphemous%5:00:00:dirty:02","synset_id":"blasphemous.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blasphemous%5:00:00:irreverent:00","call_ids":["senseval2.d002.s004.t010__v1__a1","senseval2.d002.s004.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blasphemous%5:00:00:irreverent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":1569,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066905,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.0006276,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.177710666961502}
{"item_id":"senseval2.d002.s004.t011","gold_sense_keys":["challenge%2:32:01::"],"candidates":[{"index":1,"sense_key":"challenge%2:32:01::","synset_id":"challenge.v.01"},{"index":2,"sense_key":"challenge%2:32:00::","synset_id":"challenge.v.02"},{"index":3,"sense_key":"challenge%2:32:03::","synset_id":"challenge.v.03"},{"index":4,"sense_key":"challenge%2:32:02::","synset_id":"challenge.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%2:32:01::","call_ids":["senseval2.d002.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020544999999999998,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7838786669890396}
{"item_id":"senseval2.d002.s005.t000","gold_sense_keys":["here%4:02:02::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"here%4:02:02::","call_ids":["senseval2.d002.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"here%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002855,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.726753249997273}
{"item_id":"senseval2.d002.s005.t001","gold_sense_keys":["example%1:09:00::"],"candidates":[{"index":1,"sense_key":"example%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"example%1:09:02::","synset_id":"model.n.07"},{"index":3,"sense_key":"example%1:09:01::","synset_id":"exemplar.n.01"},{"index":4,"sense_key":"example%1:10:00::","synset_id":"example.n.04"},{"index":5,"sense_key":"example%1:11:00::","synset_id":"case.n.01"},{"index":6,"sense_key":"example%1:04:00::","synset_id":"exercise.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"example%1:09:00::","call_ids":["senseval2.d002.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"example%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006755,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.469113125000149}
{"item_id":"senseval2.d002.s006.t000","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ask%2:32:02::","call_ids":["senseval2.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ask%2:32:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003446,"input_uncached_usd":0.000031,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.352870833012275}
{"item_id":"senseval2.d002.s006.t001","gold_sense_keys":["sample%1:09:00::","sample%1:09:01::"],"candidates":[{"index":1,"sense_key":"sample%1:09:00::","synset_id":"sample.n.01"},{"index":2,"sense_key":"sample%1:09:01::","synset_id":"sample_distribution.n.01"},{"index":3,"sense_key":"sample%1:17:00::","synset_id":"sample.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sample%1:09:00::","call_ids":["senseval2.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sample%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028284999999999996,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.599419875012245}
{"item_id":"senseval2.d002.s006.t002","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s006.t002__v1__a1","senseval2.d002.s006.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":544,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025905,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.00021759999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.516193041054066}
{"item_id":"senseval2.d002.s006.t003","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048639999999999995,"input_uncached_usd":0.0000192,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.142954667040613}
{"item_id":"senseval2.d002.s006.t006","gold_sense_keys":["encourage%2:41:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":2,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"encourage%2:41:00::","call_ids":["senseval2.d002.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"encourage%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018094999999999998,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9704321249737404}
{"item_id":"senseval2.d002.s006.t008","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031205,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.619782624999061}
{"item_id":"senseval2.d002.s006.t009","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"answer%1:10:00::","call_ids":["senseval2.d002.s006.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"answer%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005989000000000001,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.400865334027912}
{"item_id":"senseval2.d002.s007.t000","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mean%2:32:03::","call_ids":["senseval2.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mean%2:32:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039525,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.09618508303538}
{"item_id":"senseval2.d002.s008.t000","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007183000000000001,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.750280708016362}
{"item_id":"senseval2.d002.s009.t002","gold_sense_keys":["equate%2:31:00::"],"candidates":[{"index":1,"sense_key":"equate%2:31:00::","synset_id":"compare.v.03"},{"index":2,"sense_key":"equate%2:42:00::","synset_id":"equate.v.02"},{"index":3,"sense_key":"equate%2:30:00::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equate%2:31:00::","call_ids":["senseval2.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046389999999999995,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.124323291995097}
{"item_id":"senseval2.d002.s009.t004","gold_sense_keys":["encourage%2:41:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":2,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"encourage%2:37:00::","call_ids":["senseval2.d002.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"encourage%2:37:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.000129,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.428702166012954}
{"item_id":"senseval2.d002.s010.t000","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["senseval2.d002.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018695,"input_uncached_usd":0.000026949999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.029184625018388}
{"item_id":"senseval2.d002.s010.t001","gold_sense_keys":["generation%1:14:01::"],"candidates":[{"index":1,"sense_key":"generation%1:14:01::","synset_id":"coevals.n.01"},{"index":2,"sense_key":"generation%1:14:00::","synset_id":"generation.n.02"},{"index":3,"sense_key":"generation%1:28:00::","synset_id":"generation.n.03"},{"index":4,"sense_key":"generation%1:28:01::","synset_id":"generation.n.04"},{"index":5,"sense_key":"generation%1:11:00::","synset_id":"genesis.n.01"},{"index":6,"sense_key":"generation%1:04:01::","synset_id":"generation.n.06"},{"index":7,"sense_key":"generation%1:04:00::","synset_id":"generation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generation%1:14:01::","call_ids":["senseval2.d002.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generation%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001881,"input_uncached_usd":0.0000281,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9308054159628227}
{"item_id":"senseval2.d002.s010.t002","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016194999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.665284042013809}
{"item_id":"senseval2.d002.s010.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021094999999999998,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.727166666998528}
{"item_id":"senseval2.d002.s010.t006","gold_sense_keys":["incompetence%1:07:00::"],"candidates":[{"index":1,"sense_key":"incompetence%1:07:00::","synset_id":"incompetence.n.01"},{"index":2,"sense_key":"incompetence%1:26:00::","synset_id":"incompetence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incompetence%1:07:00::","call_ids":["senseval2.d002.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incompetence%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002563,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.831106708035804}
{"item_id":"senseval2.d002.s010.t007","gold_sense_keys":["match%2:42:01::"],"candidates":[{"index":1,"sense_key":"match%2:42:00::","synset_id":"match.v.01"},{"index":2,"sense_key":"match%2:40:00::","synset_id":"match.v.02"},{"index":3,"sense_key":"match%2:35:08::","synset_id":"match.v.03"},{"index":4,"sense_key":"match%2:42:01::","synset_id":"equal.v.02"},{"index":5,"sense_key":"match%2:30:01::","synset_id":"match.v.05"},{"index":6,"sense_key":"match%2:41:00::","synset_id":"meet.v.06"},{"index":7,"sense_key":"match%2:35:10::","synset_id":"match.v.07"},{"index":8,"sense_key":"match%2:33:00::","synset_id":"pit.v.01"},{"index":9,"sense_key":"match%2:30:00::","synset_id":"match.v.09"},{"index":10,"sense_key":"match%2:30:07::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"match%2:35:08::","call_ids":["senseval2.d002.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"match%2:35:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006595500000000001,"input_uncached_usd":0.00003875,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.732563542027492}
{"item_id":"senseval2.d002.s010.t008","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["senseval2.d002.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042125,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.64682995795738}
{"item_id":"senseval2.d002.s010.t009","gold_sense_keys":["good%3:00:01::"],"candidates":[{"index":1,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":2,"sense_key":"good%5:00:00:ample:00","synset_id":"full.s.04"},{"index":3,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":4,"sense_key":"good%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":5,"sense_key":"good%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":6,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":7,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":8,"sense_key":"good%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":9,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":10,"sense_key":"good%5:00:00:close:02","synset_id":"dear.s.02"},{"index":11,"sense_key":"good%5:00:01:sound:00","synset_id":"dependable.s.03"},{"index":12,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":13,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":14,"sense_key":"good%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":15,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":16,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":17,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":18,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":19,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":20,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":21,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"good%5:00:00:nice:00","call_ids":["senseval2.d002.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"good%5:00:00:nice:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1209,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00050245,"input_uncached_usd":0.00006045,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5706837920006365}
{"item_id":"senseval2.d002.s010.t010","gold_sense_keys":["opinion%1:09:00::"],"candidates":[{"index":1,"sense_key":"opinion%1:09:00::","synset_id":"opinion.n.01"},{"index":2,"sense_key":"opinion%1:10:01::","synset_id":"opinion.n.02"},{"index":3,"sense_key":"opinion%1:09:01::","synset_id":"public_opinion.n.01"},{"index":4,"sense_key":"opinion%1:10:00::","synset_id":"opinion.n.04"},{"index":5,"sense_key":"opinion%1:04:01::","synset_id":"opinion.n.05"},{"index":6,"sense_key":"opinion%1:09:02::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"opinion%1:09:00::","call_ids":["senseval2.d002.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"opinion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031985,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.068409500003327}
{"item_id":"senseval2.d002.s011.t000","gold_sense_keys":["whole%3:00:00::"],"candidates":[{"index":1,"sense_key":"whole%3:00:00::","synset_id":"whole.a.01"},{"index":2,"sense_key":"whole%3:00:02::","synset_id":"whole.a.02"},{"index":3,"sense_key":"whole%5:00:00:uninjured:00","synset_id":"unharmed.s.01"},{"index":4,"sense_key":"whole%5:00:00:healthy:00","synset_id":"hale.s.01"},{"index":5,"sense_key":"whole%5:00:00:undiversified:00","synset_id":"solid.s.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whole%3:00:00::","call_ids":["senseval2.d002.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whole%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028974999999999997,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.753819625009783}
{"item_id":"senseval2.d002.s011.t001","gold_sense_keys":["notion%1:09:00::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notion%1:09:00::","call_ids":["senseval2.d002.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028819999999999996,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.2805241660098545}
{"item_id":"senseval2.d002.s011.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036944999999999995,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.371741792012472}
{"item_id":"senseval2.d002.s011.t004","gold_sense_keys":["romantic%5:00:00:impractical:00"],"candidates":[{"index":1,"sense_key":"romantic%3:01:00::","synset_id":"romantic.a.01"},{"index":2,"sense_key":"romantic%5:00:01:loving:00","synset_id":"amatory.s.01"},{"index":3,"sense_key":"romantic%5:00:00:impractical:00","synset_id":"quixotic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"romantic%5:00:00:impractical:00","call_ids":["senseval2.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"romantic%5:00:00:impractical:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010824999999999999,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3467936249799095}
{"item_id":"senseval2.d002.s011.t005","gold_sense_keys":["rebellion%1:04:01::"],"candidates":[{"index":1,"sense_key":"rebellion%1:04:01::","synset_id":"rebellion.n.01"},{"index":2,"sense_key":"rebellion%1:04:00::","synset_id":"rebellion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rebellion%1:04:01::","call_ids":["senseval2.d002.s011.t005__v1__a1","senseval2.d002.s011.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rebellion%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":915,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005193499999999999,"input_uncached_usd":0.00004575,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.706061624980066}
{"item_id":"senseval2.d002.s011.t006","gold_sense_keys":["disciplined%5:00:00:controlled:00"],"candidates":[{"index":1,"sense_key":"disciplined%5:00:00:controlled:00","synset_id":"disciplined.s.01"},{"index":2,"sense_key":"disciplined%5:00:00:trained:00","synset_id":"disciplined.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"disciplined%5:00:00:trained:00","call_ids":["senseval2.d002.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"disciplined%5:00:00:trained:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038534999999999996,"input_uncached_usd":0.000020549999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.220179542026017}
{"item_id":"senseval2.d002.s011.t007","gold_sense_keys":["instruction%1:04:01::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"instruction%1:04:01::","call_ids":["senseval2.d002.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"instruction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018639999999999998,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7544386250083335}
{"item_id":"senseval2.d002.s011.t009","gold_sense_keys":["authoritarian%5:00:00:domineering:00"],"candidates":[{"index":1,"sense_key":"authoritarian%5:00:00:undemocratic:00","synset_id":"authoritarian.s.01"},{"index":2,"sense_key":"authoritarian%5:00:00:domineering:00","synset_id":"authoritarian.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"authoritarian%5:00:00:undemocratic:00","call_ids":["senseval2.d002.s011.t009__v1__a1","senseval2.d002.s011.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"authoritarian%5:00:00:undemocratic:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":957,"cached_input_tokens":0,"output_tokens":2720,"reasoning_output_tokens":2688},"cost":{"currency":"USD","total_usd":0.0011358499999999999,"input_uncached_usd":0.00004785,"input_cached_usd":0.0,"output_usd":0.001088,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.289501415973064}
{"item_id":"senseval2.d002.s011.t010","gold_sense_keys":["repression%1:04:00::"],"candidates":[{"index":1,"sense_key":"repression%1:26:00::","synset_id":"repression.n.01"},{"index":2,"sense_key":"repression%1:22:00::","synset_id":"repression.n.02"},{"index":3,"sense_key":"repression%1:04:00::","synset_id":"repression.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"repression%1:04:00::","call_ids":["senseval2.d002.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"repression%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001827,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.93871587497415}
{"item_id":"senseval2.d002.s011.t011","gold_sense_keys":["frustration%1:04:02::"],"candidates":[{"index":1,"sense_key":"frustration%1:12:00::","synset_id":"frustration.n.01"},{"index":2,"sense_key":"frustration%1:04:02::","synset_id":"frustration.n.02"},{"index":3,"sense_key":"frustration%1:12:01::","synset_id":"frustration.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"frustration%1:04:02::","call_ids":["senseval2.d002.s011.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"frustration%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004132,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.626273957954254}
{"item_id":"senseval2.d002.s011.t012","gold_sense_keys":["latent%5:00:00:potential:00"],"candidates":[{"index":1,"sense_key":"latent%5:00:00:potential:00","synset_id":"latent.s.01"},{"index":2,"sense_key":"latent%5:00:00:inactive:02","synset_id":"latent.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"latent%5:00:00:potential:00","call_ids":["senseval2.d002.s011.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"latent%5:00:00:potential:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006160499999999999,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.350316958967596}
{"item_id":"senseval2.d002.s011.t013","gold_sense_keys":["talent%1:09:00::"],"candidates":[{"index":1,"sense_key":"talent%1:09:00::","synset_id":"endowment.n.01"},{"index":2,"sense_key":"talent%1:18:00::","synset_id":"talent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talent%1:09:00::","call_ids":["senseval2.d002.s011.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talent%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002824,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.353081542009022}
{"item_id":"senseval2.d002.s011.t016","gold_sense_keys":["yet%4:02:02::"],"candidates":[{"index":1,"sense_key":"yet%4:02:02::","synset_id":"yet.r.01"},{"index":2,"sense_key":"yet%4:02:04::","synset_id":"so_far.r.01"},{"index":3,"sense_key":"yet%4:02:03::","synset_id":"even.r.03"},{"index":4,"sense_key":"yet%4:02:00::","synset_id":"yet.r.04"},{"index":5,"sense_key":"yet%4:02:05::","synset_id":"so_far.r.03"},{"index":6,"sense_key":"yet%4:02:06::","synset_id":"however.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"yet%4:02:02::","call_ids":["senseval2.d002.s011.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"yet%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":1809,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007574999999999999,"input_uncached_usd":0.0000339,"input_cached_usd":0.0,"output_usd":0.0007235999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.738799415994436}
{"item_id":"senseval2.d002.s011.t018","gold_sense_keys":["potentiality%1:09:00::"],"candidates":[{"index":1,"sense_key":"potentiality%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potentiality%1:09:00::","synset_id":"capability.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potentiality%1:09:00::","call_ids":["senseval2.d002.s011.t018__v1__a1","senseval2.d002.s011.t018__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potentiality%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":855,"cached_input_tokens":0,"output_tokens":2400,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.0010027500000000002,"input_uncached_usd":0.000042749999999999996,"input_cached_usd":0.0,"output_usd":0.0009599999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.704174124926794}
{"item_id":"senseval2.d002.s011.t019","gold_sense_keys":["inherent%5:00:00:intrinsic:00"],"candidates":[{"index":1,"sense_key":"inherent%5:00:00:intrinsic:00","synset_id":"built-in.s.01"},{"index":2,"sense_key":"inherent%5:00:00:implicit:00","synset_id":"implicit_in.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inherent%5:00:00:intrinsic:00","call_ids":["senseval2.d002.s011.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inherent%5:00:00:intrinsic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006434500000000001,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.827373666979838}
{"item_id":"senseval2.d002.s011.t020","gold_sense_keys":["soul%1:18:00::"],"candidates":[{"index":1,"sense_key":"soul%1:18:00::","synset_id":"soul.n.01"},{"index":2,"sense_key":"soul%1:03:00::","synset_id":"person.n.01"},{"index":3,"sense_key":"soul%1:12:00::","synset_id":"soul.n.03"},{"index":4,"sense_key":"soul%1:04:00::","synset_id":"soul.n.04"},{"index":5,"sense_key":"soul%1:10:00::","synset_id":"soul.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"soul%1:18:00::","call_ids":["senseval2.d002.s011.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"soul%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004167,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.130173332989216}
{"item_id":"senseval2.d002.s011.t021","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s011.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018554999999999998,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5210297500016168}
{"item_id":"senseval2.d002.s012.t001","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015455,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3096492079785094}
{"item_id":"senseval2.d002.s012.t002","gold_sense_keys":["find%2:31:10::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"find%2:31:10::","call_ids":["senseval2.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"find%2:31:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1118,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002671,"input_uncached_usd":0.0000559,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4890579999773763}
{"item_id":"senseval2.d002.s012.t003","gold_sense_keys":["romantic%5:00:00:impractical:00"],"candidates":[{"index":1,"sense_key":"romantic%3:01:00::","synset_id":"romantic.a.01"},{"index":2,"sense_key":"romantic%5:00:01:loving:00","synset_id":"amatory.s.01"},{"index":3,"sense_key":"romantic%5:00:00:impractical:00","synset_id":"quixotic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"romantic%5:00:00:impractical:00","call_ids":["senseval2.d002.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"romantic%5:00:00:impractical:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013324999999999999,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0843153329915367}
{"item_id":"senseval2.d002.s012.t004","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027725,"input_uncached_usd":0.00004045,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.685756041959394}
{"item_id":"senseval2.d002.s012.t005","gold_sense_keys":["attractive%5:00:00:appealing:00"],"candidates":[{"index":1,"sense_key":"attractive%3:00:01::","synset_id":"attractive.a.01"},{"index":2,"sense_key":"attractive%5:00:00:appealing:00","synset_id":"attractive.s.01"},{"index":3,"sense_key":"attractive%3:00:02::","synset_id":"attractive.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attractive%5:00:00:appealing:00","call_ids":["senseval2.d002.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attractive%5:00:00:appealing:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020794999999999999,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.331225165980868}
{"item_id":"senseval2.d002.s013.t001","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval2.d002.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036369999999999995,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.195306583016645}
{"item_id":"senseval2.d002.s013.t002","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043624999999999995,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.781280958966818}
{"item_id":"senseval2.d002.s013.t003","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026304999999999997,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.114365333982278}
{"item_id":"senseval2.d002.s013.t004","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021109999999999998,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.555998958996497}
{"item_id":"senseval2.d002.s013.t005","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1961,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00061645,"input_uncached_usd":0.00009805,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.81655295897508}
{"item_id":"senseval2.d002.s013.t006","gold_sense_keys":["decent%5:00:00:sufficient:00"],"candidates":[{"index":1,"sense_key":"decent%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":2,"sense_key":"decent%5:00:00:proper:00","synset_id":"becoming.s.01"},{"index":3,"sense_key":"decent%3:00:00::","synset_id":"decent.a.03"},{"index":4,"sense_key":"decent%5:00:00:sufficient:00","synset_id":"adequate.s.01"},{"index":5,"sense_key":"decent%5:00:02:modest:01","synset_id":"decent.s.04"},{"index":6,"sense_key":"decent%5:00:00:modest:01","synset_id":"decent.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decent%5:00:00:proper:00","call_ids":["senseval2.d002.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decent%5:00:00:proper:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005492,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.864326874958351}
{"item_id":"senseval2.d002.s013.t007","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"education%1:09:00::","call_ids":["senseval2.d002.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"education%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044615,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.5622612909646705}
{"item_id":"senseval2.d002.s013.t013","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s013.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038134999999999997,"input_uncached_usd":0.00004215,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.733520124980714}
{"item_id":"senseval2.d002.s013.t014","gold_sense_keys":["demand%2:42:00::"],"candidates":[{"index":1,"sense_key":"demand%2:32:00::","synset_id":"demand.v.01"},{"index":2,"sense_key":"demand%2:42:00::","synset_id":"necessitate.v.01"},{"index":3,"sense_key":"demand%2:32:02::","synset_id":"demand.v.03"},{"index":4,"sense_key":"demand%2:32:09::","synset_id":"demand.v.04"},{"index":5,"sense_key":"demand%2:32:05::","synset_id":"demand.v.05"},{"index":6,"sense_key":"demand%2:32:06::","synset_id":"demand.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"demand%2:42:00::","call_ids":["senseval2.d002.s013.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"demand%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000289,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.566998832975514}
{"item_id":"senseval2.d002.s014.t000","gold_sense_keys":["commitment%1:04:00::"],"candidates":[{"index":1,"sense_key":"commitment%1:07:01::","synset_id":"committedness.n.01"},{"index":2,"sense_key":"commitment%1:04:00::","synset_id":"commitment.n.02"},{"index":3,"sense_key":"commitment%1:04:02::","synset_id":"commitment.n.03"},{"index":4,"sense_key":"commitment%1:10:00::","synset_id":"commitment.n.04"},{"index":5,"sense_key":"commitment%1:04:01::","synset_id":"commitment.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"commitment%1:04:02::","call_ids":["senseval2.d002.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"commitment%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":1490,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006245,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.000596,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.952236499986611}
{"item_id":"senseval2.d002.s014.t002","gold_sense_keys":["survive%2:42:02::"],"candidates":[{"index":1,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survive%2:42:01::","call_ids":["senseval2.d002.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survive%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002653,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.551110166998114}
{"item_id":"senseval2.d002.s014.t003","gold_sense_keys":["adolescent%3:01:00::","adolescent%5:00:00:immature:01"],"candidates":[{"index":1,"sense_key":"adolescent%3:01:00::","synset_id":"adolescent.a.01"},{"index":2,"sense_key":"adolescent%5:00:00:young:00","synset_id":"adolescent.s.01"},{"index":3,"sense_key":"adolescent%5:00:00:immature:02","synset_id":"adolescent.s.02"},{"index":4,"sense_key":"adolescent%5:00:00:immature:01","synset_id":"adolescent.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"adolescent%5:00:00:young:00","call_ids":["senseval2.d002.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"adolescent%5:00:00:young:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036619999999999996,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.706820041988976}
{"item_id":"senseval2.d002.s014.t004","gold_sense_keys":["illiteracy%1:09:00::"],"candidates":[{"index":1,"sense_key":"illiteracy%1:09:01::","synset_id":"illiteracy.n.01"},{"index":2,"sense_key":"illiteracy%1:09:00::","synset_id":"illiteracy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"illiteracy%1:09:00::","call_ids":["senseval2.d002.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"illiteracy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006414,"input_uncached_usd":0.0000206,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.903964624973014}
{"item_id":"senseval2.d002.s015.t000","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s015.t000__v1__a1","senseval2.d002.s015.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":901,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00059545,"input_uncached_usd":0.00004505,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.417634750949219}
{"item_id":"senseval2.d002.s015.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00080495,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.924124707991723}
{"item_id":"senseval2.d002.s015.t002","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":2,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%1:28:00::","call_ids":["senseval2.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015654999999999998,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0944608750287443}
{"item_id":"senseval2.d002.s015.t003","gold_sense_keys":["determine%2:31:02::"],"candidates":[{"index":1,"sense_key":"determine%2:32:00::","synset_id":"determine.v.01"},{"index":2,"sense_key":"determine%2:31:02::","synset_id":"determine.v.02"},{"index":3,"sense_key":"determine%2:31:01::","synset_id":"determine.v.03"},{"index":4,"sense_key":"determine%2:32:03::","synset_id":"specify.v.02"},{"index":5,"sense_key":"determine%2:31:00::","synset_id":"decide.v.01"},{"index":6,"sense_key":"determine%2:31:03::","synset_id":"determine.v.06"},{"index":7,"sense_key":"determine%2:32:05::","synset_id":"settle.v.03"},{"index":8,"sense_key":"determine%2:32:01::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"determine%2:32:03::","call_ids":["senseval2.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"determine%2:32:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042685,"input_uncached_usd":0.00003645,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.205770083994139}
{"item_id":"senseval2.d002.s015.t004","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"degree%1:07:00::","call_ids":["senseval2.d002.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004228,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.752643084037118}
{"item_id":"senseval2.d002.s015.t005","gold_sense_keys":["allow%2:41:00::","allow%2:42:00::"],"candidates":[{"index":1,"sense_key":"allow%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"allow%2:32:00::","synset_id":"permit.v.01"},{"index":3,"sense_key":"allow%2:40:00::","synset_id":"allow.v.03"},{"index":4,"sense_key":"allow%2:31:01::","synset_id":"allow.v.04"},{"index":5,"sense_key":"allow%2:42:00::","synset_id":"leave.v.06"},{"index":6,"sense_key":"allow%2:31:00::","synset_id":"allow.v.06"},{"index":7,"sense_key":"allow%2:42:07::","synset_id":"admit.v.05"},{"index":8,"sense_key":"allow%2:41:01::","synset_id":"give_up.v.11"},{"index":9,"sense_key":"allow%2:40:01::","synset_id":"allow.v.09"},{"index":10,"sense_key":"allow%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"allow%2:42:00::","call_ids":["senseval2.d002.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"allow%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066325,"input_uncached_usd":0.000042449999999999995,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.37019154097652}
{"item_id":"senseval2.d002.s015.t007","gold_sense_keys":["prevail%2:33:00::"],"candidates":[{"index":1,"sense_key":"prevail%2:42:02::","synset_id":"predominate.v.01"},{"index":2,"sense_key":"prevail%2:42:01::","synset_id":"prevail.v.02"},{"index":3,"sense_key":"prevail%2:42:00::","synset_id":"prevail.v.03"},{"index":4,"sense_key":"prevail%2:33:00::","synset_id":"prevail.v.04"},{"index":5,"sense_key":"prevail%2:32:02::","synset_id":"prevail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"prevail%2:33:00::","call_ids":["senseval2.d002.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"prevail%2:33:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029035,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.340600249997806}
{"item_id":"senseval2.d002.s015.t008","gold_sense_keys":["illusion%1:09:00::"],"candidates":[{"index":1,"sense_key":"illusion%1:09:01::","synset_id":"illusion.n.01"},{"index":2,"sense_key":"illusion%1:09:00::","synset_id":"illusion.n.02"},{"index":3,"sense_key":"illusion%1:04:00::","synset_id":"delusion.n.03"},{"index":4,"sense_key":"illusion%1:04:02::","synset_id":"magic_trick.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"illusion%1:04:00::","call_ids":["senseval2.d002.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"illusion%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049235,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.958733041014057}
{"item_id":"senseval2.d002.s015.t010","gold_sense_keys":["share%2:42:00::"],"candidates":[{"index":1,"sense_key":"share%2:42:00::","synset_id":"share.v.01"},{"index":2,"sense_key":"share%2:40:02::","synset_id":"share.v.02"},{"index":3,"sense_key":"share%2:40:01::","synset_id":"partake.v.02"},{"index":4,"sense_key":"share%2:40:00::","synset_id":"share.v.04"},{"index":5,"sense_key":"share%2:32:00::","synset_id":"share.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"share%2:42:00::","call_ids":["senseval2.d002.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"share%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001344,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2447883329587057}
{"item_id":"senseval2.d002.s016.t000","gold_sense_keys":["education%1:04:00::","education%1:04:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"education%1:09:00::","call_ids":["senseval2.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"education%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009325000000000001,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.176805290975608}
{"item_id":"senseval2.d002.s016.t001","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":3,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":4,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":5,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":6,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":7,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"establishment%1:14:02::","call_ids":["senseval2.d002.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"establishment%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031925,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.039334291999694}
{"item_id":"senseval2.d002.s016.t002","gold_sense_keys":["fight%2:33:01::"],"candidates":[{"index":1,"sense_key":"fight%2:33:00::","synset_id":"contend.v.06"},{"index":2,"sense_key":"fight%2:33:01::","synset_id":"fight.v.02"},{"index":3,"sense_key":"fight%2:41:00::","synset_id":"fight.v.03"},{"index":4,"sense_key":"fight%2:41:10::","synset_id":"crusade.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fight%2:33:01::","call_ids":["senseval2.d002.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fight%2:33:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001368,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.707889083016198}
{"item_id":"senseval2.d002.s017.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reason%1:10:00::","call_ids":["senseval2.d002.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005987,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.470886208990123}
{"item_id":"senseval2.d002.s017.t002","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"simple%5:00:00:easy:01","call_ids":["senseval2.d002.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"simple%5:00:00:easy:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054825,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.919450625020545}
{"item_id":"senseval2.d002.s017.t003","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reason%1:10:00::","call_ids":["senseval2.d002.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004958499999999999,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.750768750032876}
{"item_id":"senseval2.d002.s018.t000","gold_sense_keys":["progressive%3:00:01::"],"candidates":[{"index":1,"sense_key":"progressive%3:00:01::","synset_id":"progressive.a.01"},{"index":2,"sense_key":"progressive%5:00:00:liberal:00","synset_id":"progressive.s.01"},{"index":3,"sense_key":"progressive%3:00:02::","synset_id":"progressive.a.03"},{"index":4,"sense_key":"progressive%5:00:00:increasing:00","synset_id":"progressive.s.02"},{"index":5,"sense_key":"progressive%5:00:00:ordered:00","synset_id":"progressive.s.03"},{"index":6,"sense_key":"progressive%5:00:00:active:02","synset_id":"progressive.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"progressive%3:00:01::","call_ids":["senseval2.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"progressive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006726,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.222398167010397}
{"item_id":"senseval2.d002.s018.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":2640,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.0010848,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0010559999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.01199374999851}
{"item_id":"senseval2.d002.s018.t002","gold_sense_keys":["once%4:02:01::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"once%4:02:01::","call_ids":["senseval2.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"once%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010444999999999999,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.17894808400888}
{"item_id":"senseval2.d002.s018.t003","gold_sense_keys":["call%2:32:02::"],"candidates":[{"index":1,"sense_key":"call%2:32:02::","synset_id":"name.v.01"},{"index":2,"sense_key":"call%2:32:00::","synset_id":"call.v.02"},{"index":3,"sense_key":"call%2:32:01::","synset_id":"call.v.03"},{"index":4,"sense_key":"call%2:32:09::","synset_id":"shout.v.02"},{"index":5,"sense_key":"call%2:32:05::","synset_id":"call.v.05"},{"index":6,"sense_key":"call%2:41:00::","synset_id":"visit.v.03"},{"index":7,"sense_key":"call%2:41:04::","synset_id":"call.v.07"},{"index":8,"sense_key":"call%2:32:13::","synset_id":"call.v.08"},{"index":9,"sense_key":"call%2:32:04::","synset_id":"call.v.09"},{"index":10,"sense_key":"call%2:32:03::","synset_id":"call.v.10"},{"index":11,"sense_key":"call%2:42:09::","synset_id":"call.v.11"},{"index":12,"sense_key":"call%2:41:14::","synset_id":"address.v.06"},{"index":13,"sense_key":"call%2:41:12::","synset_id":"call.v.13"},{"index":14,"sense_key":"call%2:40:09::","synset_id":"call.v.14"},{"index":15,"sense_key":"call%2:40:05::","synset_id":"bid.v.04"},{"index":16,"sense_key":"call%2:38:00::","synset_id":"call.v.16"},{"index":17,"sense_key":"call%2:32:06::","synset_id":"call.v.17"},{"index":18,"sense_key":"call%2:32:10::","synset_id":"predict.v.01"},{"index":19,"sense_key":"call%2:32:12::","synset_id":"call.v.19"},{"index":20,"sense_key":"call%2:32:07::","synset_id":"call.v.20"},{"index":21,"sense_key":"call%2:32:08::","synset_id":"call.v.21"},{"index":22,"sense_key":"call%2:32:11::","synset_id":"call.v.22"},{"index":23,"sense_key":"call%2:32:15::","synset_id":"call.v.23"},{"index":24,"sense_key":"call%2:32:14::","synset_id":"call.v.24"},{"index":25,"sense_key":"call%2:31:13::","synset_id":"call.v.25"},{"index":26,"sense_key":"call%2:31:00::","synset_id":"call.v.26"},{"index":27,"sense_key":"call%2:31:05::","synset_id":"call.v.27"},{"index":28,"sense_key":"call%2:29:05::","synset_id":"call.v.28"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call%2:32:02::","call_ids":["senseval2.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1486,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00028549999999999995,"input_uncached_usd":0.00007429999999999999,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9189827920054086}
{"item_id":"senseval2.d002.s018.t004","gold_sense_keys":["far%4:02:02::"],"candidates":[{"index":1,"sense_key":"far%4:02:02::","synset_id":"far.r.01"},{"index":2,"sense_key":"far%4:02:00::","synset_id":"far.r.02"},{"index":3,"sense_key":"far%4:02:04::","synset_id":"far.r.03"},{"index":4,"sense_key":"far%4:02:01::","synset_id":"far.r.04"},{"index":5,"sense_key":"far%4:02:05::","synset_id":"far.r.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"far%4:02:02::","call_ids":["senseval2.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"far%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021155,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4668352080043405}
{"item_id":"senseval2.d002.s018.t005","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["senseval2.d002.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033329999999999997,"input_uncached_usd":0.000019699999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.213902125018649}
{"item_id":"senseval2.d002.s018.t007","gold_sense_keys":["agreeable%3:00:00::"],"candidates":[{"index":1,"sense_key":"agreeable%3:00:00::","synset_id":"agreeable.a.01"},{"index":2,"sense_key":"agreeable%5:00:00:consistent:00","synset_id":"accordant.s.01"},{"index":3,"sense_key":"agreeable%5:00:00:accordant:00","synset_id":"agreeable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"agreeable%5:00:00:consistent:00","call_ids":["senseval2.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"agreeable%5:00:00:consistent:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031035,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.480419999978039}
{"item_id":"senseval2.d002.s018.t008","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017849999999999997,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1006472919834778}
{"item_id":"senseval2.d002.s018.t009","gold_sense_keys":["disciplined%5:00:00:controlled:00"],"candidates":[{"index":1,"sense_key":"disciplined%5:00:00:controlled:00","synset_id":"disciplined.s.01"},{"index":2,"sense_key":"disciplined%5:00:00:trained:00","synset_id":"disciplined.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"disciplined%5:00:00:controlled:00","call_ids":["senseval2.d002.s018.t009__v1__a1","senseval2.d002.s018.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"disciplined%5:00:00:controlled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":1952,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008213499999999999,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.0007808,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.063245251018088}
{"item_id":"senseval2.d002.s018.t010","gold_sense_keys":["instruction%1:04:01::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"instruction%1:04:01::","call_ids":["senseval2.d002.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"instruction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018495,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.867465417017229}
{"item_id":"senseval2.d002.s019.t000","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"},{"index":2,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":5,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nice%3:00:00::","call_ids":["senseval2.d002.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002395,"input_uncached_usd":0.0000283,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.771075959026348}
{"item_id":"senseval2.d002.s019.t001","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s019.t001__v1__a1","senseval2.d002.s019.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048715,"input_uncached_usd":0.00003915,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.97717512503732}
{"item_id":"senseval2.d002.s019.t002","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"think%2:31:01::","call_ids":["senseval2.d002.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"think%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":875,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005109499999999999,"input_uncached_usd":0.00004375,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.608147541002836}
{"item_id":"senseval2.d002.s019.t003","gold_sense_keys":["personality%1:07:00::"],"candidates":[{"index":1,"sense_key":"personality%1:07:00::","synset_id":"personality.n.01"},{"index":2,"sense_key":"personality%1:18:00::","synset_id":"personality.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"personality%1:07:00::","call_ids":["senseval2.d002.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"personality%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015414999999999998,"input_uncached_usd":0.00001975,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.832188041997142}
{"item_id":"senseval2.d002.s019.t004","gold_sense_keys":["development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d002.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006088,"input_uncached_usd":0.0000392,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.98969041701639}
{"item_id":"senseval2.d002.s019.t005","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013245,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.405624583014287}
{"item_id":"senseval2.d002.s019.t006","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"},{"index":2,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":5,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nice%3:00:00::","call_ids":["senseval2.d002.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018835,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9160357079817913}
{"item_id":"senseval2.d002.s019.t007","gold_sense_keys":["minimize%2:30:00::"],"candidates":[{"index":1,"sense_key":"minimize%2:30:00::","synset_id":"minimize.v.01"},{"index":2,"sense_key":"minimize%2:32:01::","synset_id":"understate.v.01"},{"index":3,"sense_key":"minimize%2:32:00::","synset_id":"minimize.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minimize%2:30:00::","call_ids":["senseval2.d002.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minimize%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028325,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.532395332993474}
{"item_id":"senseval2.d002.s019.t009","gold_sense_keys":["test%1:10:00::"],"candidates":[{"index":1,"sense_key":"test%1:09:02::","synset_id":"trial.n.02"},{"index":2,"sense_key":"test%1:04:02::","synset_id":"test.n.02"},{"index":3,"sense_key":"test%1:10:00::","synset_id":"examination.n.02"},{"index":4,"sense_key":"test%1:04:03::","synset_id":"test.n.04"},{"index":5,"sense_key":"test%1:04:00::","synset_id":"test.n.05"},{"index":6,"sense_key":"test%1:05:00::","synset_id":"test.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"test%1:10:00::","call_ids":["senseval2.d002.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"test%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011274999999999999,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1693156660185196}
{"item_id":"senseval2.d002.s019.t010","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d002.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041170000000000003,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.177054999978282}
{"item_id":"senseval2.d002.s019.t012","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["senseval2.d002.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005954,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.651604750019033}
{"item_id":"senseval2.d002.s020.t001","gold_sense_keys":["provide%2:40:00::"],"candidates":[{"index":1,"sense_key":"provide%2:40:00::","synset_id":"supply.v.01"},{"index":2,"sense_key":"provide%2:34:00::","synset_id":"provide.v.02"},{"index":3,"sense_key":"provide%2:32:00::","synset_id":"provide.v.03"},{"index":4,"sense_key":"provide%2:41:00::","synset_id":"put_up.v.02"},{"index":5,"sense_key":"provide%2:42:00::","synset_id":"leave.v.06"},{"index":6,"sense_key":"provide%2:40:08::","synset_id":"provide.v.06"},{"index":7,"sense_key":"provide%2:30:05::","synset_id":"provide.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"provide%2:40:00::","call_ids":["senseval2.d002.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"provide%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047469999999999994,"input_uncached_usd":0.0000331,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.563481374992989}
{"item_id":"senseval2.d002.s020.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022969999999999997,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.834709707996808}
{"item_id":"senseval2.d002.s020.t003","gold_sense_keys":["profession%1:04:00::","profession%1:14:00::"],"candidates":[{"index":1,"sense_key":"profession%1:14:00::","synset_id":"profession.n.01"},{"index":2,"sense_key":"profession%1:04:00::","synset_id":"profession.n.02"},{"index":3,"sense_key":"profession%1:10:00::","synset_id":"profession.n.03"},{"index":4,"sense_key":"profession%1:10:01::","synset_id":"profession.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"profession%1:04:00::","call_ids":["senseval2.d002.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"profession%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028514999999999997,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.011879917001352}
{"item_id":"senseval2.d002.s020.t004","gold_sense_keys":["pass%2:41:08::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"pass%2:41:08::","call_ids":["senseval2.d002.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"pass%2:41:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1448,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0003092,"input_uncached_usd":0.0000724,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.409375416988041}
{"item_id":"senseval2.d002.s020.t005","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":2,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":3,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":4,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":5,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":6,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":7,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":8,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002474,"input_uncached_usd":0.0000362,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.045128250028938}
{"item_id":"senseval2.d002.s020.t006","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043464999999999997,"input_uncached_usd":0.00001865,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.822329415997956}
{"item_id":"senseval2.d002.s020.t008","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046024999999999994,"input_uncached_usd":0.00001865,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.80620925000403}
{"item_id":"senseval2.d002.s020.t009","gold_sense_keys":["philosophy%1:09:00::"],"candidates":[{"index":1,"sense_key":"philosophy%1:09:01::","synset_id":"doctrine.n.01"},{"index":2,"sense_key":"philosophy%1:09:00::","synset_id":"philosophy.n.02"},{"index":3,"sense_key":"philosophy%1:09:02::","synset_id":"philosophy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"philosophy%1:09:00::","call_ids":["senseval2.d002.s020.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"philosophy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031025,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.9066854160046205}
{"item_id":"senseval2.d002.s021.t000","gold_sense_keys":["take%2:31:02::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":25,"chosen_sense_key":"take%2:31:02::","call_ids":["senseval2.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":25,"predicted_sense_key":"take%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2153,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00065165,"input_uncached_usd":0.00010765,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.057001375011168}
{"item_id":"senseval2.d002.s021.t002","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":2,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":3,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":4,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":5,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":6,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":7,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":8,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039999999999999996,"input_uncached_usd":0.0000352,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.672186708019581}
{"item_id":"senseval2.d002.s021.t003","gold_sense_keys":["college%1:14:01::"],"candidates":[{"index":1,"sense_key":"college%1:14:00::","synset_id":"college.n.01"},{"index":2,"sense_key":"college%1:14:01::","synset_id":"college.n.02"},{"index":3,"sense_key":"college%1:06:00::","synset_id":"college.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"college%1:14:01::","call_ids":["senseval2.d002.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"college%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":378,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040930000000000003,"input_uncached_usd":0.0000189,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.652919707994442}
{"item_id":"senseval2.d002.s021.t004","gold_sense_keys":["think%2:31:03::","think%2:36:00::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval2.d002.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009196499999999999,"input_uncached_usd":0.00004285,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.251201083010528}
{"item_id":"senseval2.d002.s022.t001","gold_sense_keys":["fairly%4:02:00::"],"candidates":[{"index":1,"sense_key":"fairly%4:02:02::","synset_id":"reasonably.r.01"},{"index":2,"sense_key":"fairly%4:02:00::","synset_id":"fairly.r.02"},{"index":3,"sense_key":"fairly%4:02:03::","synset_id":"fairly.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fairly%4:02:02::","call_ids":["senseval2.d002.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fairly%4:02:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001815,"input_uncached_usd":0.000021499999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.40721241699066}
{"item_id":"senseval2.d002.s022.t002","gold_sense_keys":["describe%2:32:00::"],"candidates":[{"index":1,"sense_key":"describe%2:32:00::","synset_id":"describe.v.01"},{"index":2,"sense_key":"describe%2:32:01::","synset_id":"report.v.01"},{"index":3,"sense_key":"describe%2:35:00::","synset_id":"trace.v.02"},{"index":4,"sense_key":"describe%2:31:00::","synset_id":"identify.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"describe%2:32:01::","call_ids":["senseval2.d002.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"describe%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023475,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.322910916002002}
{"item_id":"senseval2.d002.s022.t003","gold_sense_keys":["pap%1:10:00::"],"candidates":[{"index":1,"sense_key":"pap%1:10:00::","synset_id":"pap.n.01"},{"index":2,"sense_key":"pap%1:13:00::","synset_id":"soft_diet.n.01"},{"index":3,"sense_key":"pap%1:08:00::","synset_id":"nipple.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pap%1:10:00::","call_ids":["senseval2.d002.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pap%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017979999999999998,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4303661250160076}
{"item_id":"senseval2.d002.s022.t004","gold_sense_keys":["course%1:04:01::"],"candidates":[{"index":1,"sense_key":"course%1:04:01::","synset_id":"course.n.01"},{"index":2,"sense_key":"course%1:14:00::","synset_id":"course.n.02"},{"index":3,"sense_key":"course%1:15:00::","synset_id":"course.n.03"},{"index":4,"sense_key":"course%1:04:00::","synset_id":"course.n.04"},{"index":5,"sense_key":"course%1:17:00::","synset_id":"path.n.04"},{"index":6,"sense_key":"course%1:14:01::","synset_id":"class.n.02"},{"index":7,"sense_key":"course%1:13:00::","synset_id":"course.n.07"},{"index":8,"sense_key":"course%1:06:01::","synset_id":"course.n.08"},{"index":9,"sense_key":"course%1:06:00::","synset_id":"course.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"course%1:04:01::","call_ids":["senseval2.d002.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"course%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":704,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045119999999999996,"input_uncached_usd":0.0000352,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1117620000150055}
{"item_id":"senseval2.d002.s023.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s023.t002__v1__a1","senseval2.d002.s023.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":800,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035784999999999994,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.00031999999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4650657920283265}
{"item_id":"senseval2.d002.s023.t003","gold_sense_keys":["distinct%5:00:00:different:00"],"candidates":[{"index":1,"sense_key":"distinct%5:00:00:different:00","synset_id":"distinct.s.01"},{"index":2,"sense_key":"distinct%3:00:00::","synset_id":"distinct.a.02"},{"index":3,"sense_key":"distinct%5:00:00:separate:00","synset_id":"discrete.s.01"},{"index":4,"sense_key":"distinct%5:00:00:definite:00","synset_id":"distinct.s.03"},{"index":5,"sense_key":"distinct%5:00:00:clear:00","synset_id":"clear-cut.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"distinct%5:00:00:different:00","call_ids":["senseval2.d002.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"distinct%5:00:00:different:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021299999999999997,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.204034084046725}
{"item_id":"senseval2.d002.s023.t004","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s023.t004__v1__a1","senseval2.d002.s023.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006142499999999999,"input_uncached_usd":0.000038249999999999995,"input_cached_usd":0.0,"output_usd":0.000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.387026333017275}
{"item_id":"senseval2.d002.s023.t005","gold_sense_keys":["establishment%1:14:00::","establishment%1:14:02::"],"candidates":[{"index":1,"sense_key":"establishment%1:04:00::","synset_id":"constitution.n.02"},{"index":2,"sense_key":"establishment%1:14:00::","synset_id":"institution.n.01"},{"index":3,"sense_key":"establishment%1:14:02::","synset_id":"administration.n.02"},{"index":4,"sense_key":"establishment%1:06:00::","synset_id":"establishment.n.04"},{"index":5,"sense_key":"establishment%1:14:01::","synset_id":"establishment.n.05"},{"index":6,"sense_key":"establishment%1:22:00::","synset_id":"establishment.n.06"},{"index":7,"sense_key":"establishment%1:09:00::","synset_id":"establishment.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"establishment%1:14:02::","call_ids":["senseval2.d002.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"establishment%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":2001,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008295999999999999,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.0008003999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.27557166700717}
{"item_id":"senseval2.d002.s024.t000","gold_sense_keys":["know%2:31:00::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"know%2:31:00::","call_ids":["senseval2.d002.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"know%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020094999999999998,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.269224125018809}
{"item_id":"senseval2.d002.s024.t004","gold_sense_keys":["seriously%4:02:00::"],"candidates":[{"index":1,"sense_key":"seriously%4:02:00::","synset_id":"seriously.r.01"},{"index":2,"sense_key":"seriously%4:02:01::","synset_id":"badly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seriously%4:02:00::","call_ids":["senseval2.d002.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seriously%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012839999999999998,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5689951250096783}
{"item_id":"senseval2.d002.s024.t005","gold_sense_keys":["committed%3:00:00::"],"candidates":[{"index":1,"sense_key":"committed%3:00:00::","synset_id":"committed.a.01"},{"index":2,"sense_key":"committed%3:00:04::","synset_id":"attached.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committed%3:00:00::","call_ids":["senseval2.d002.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committed%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004858,"input_uncached_usd":0.000018599999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.273526999983005}
{"item_id":"senseval2.d002.s024.t007","gold_sense_keys":["teaching%1:04:00::","teaching%1:04:01::"],"candidates":[{"index":1,"sense_key":"teaching%1:04:00::","synset_id":"teaching.n.01"},{"index":2,"sense_key":"teaching%1:09:00::","synset_id":"teaching.n.02"},{"index":3,"sense_key":"teaching%1:04:01::","synset_id":"education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"teaching%1:04:01::","call_ids":["senseval2.d002.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"teaching%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001826,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.727581083017867}
{"item_id":"senseval2.d002.s025.t000","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"best%3:00:00::","call_ids":["senseval2.d002.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"best%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1314,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00022569999999999998,"input_uncached_usd":0.0000657,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.055407000007108}
{"item_id":"senseval2.d002.s025.t001","gold_sense_keys":["bright%5:00:00:intelligent:00"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bright%5:00:00:intelligent:00","call_ids":["senseval2.d002.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bright%5:00:00:intelligent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017219999999999998,"input_uncached_usd":0.0000378,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5157514159800485}
{"item_id":"senseval2.d002.s025.t002","gold_sense_keys":["generation%1:14:01::"],"candidates":[{"index":1,"sense_key":"generation%1:14:01::","synset_id":"coevals.n.01"},{"index":2,"sense_key":"generation%1:14:00::","synset_id":"generation.n.02"},{"index":3,"sense_key":"generation%1:28:00::","synset_id":"generation.n.03"},{"index":4,"sense_key":"generation%1:28:01::","synset_id":"generation.n.04"},{"index":5,"sense_key":"generation%1:11:00::","synset_id":"genesis.n.01"},{"index":6,"sense_key":"generation%1:04:01::","synset_id":"generation.n.06"},{"index":7,"sense_key":"generation%1:04:00::","synset_id":"generation.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"generation%1:28:00::","call_ids":["senseval2.d002.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"generation%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039024999999999997,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.790834540966898}
{"item_id":"senseval2.d002.s025.t003","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d002.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035784999999999994,"input_uncached_usd":0.00001865,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.646067875029985}
{"item_id":"senseval2.d002.s025.t006","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval2.d002.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043864999999999996,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.064539999992121}
{"item_id":"senseval2.d002.s026.t000","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"job%1:04:05::","call_ids":["senseval2.d002.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"job%1:04:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037989999999999996,"input_uncached_usd":0.0000407,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.72860979201505}
{"item_id":"senseval2.d002.s026.t001","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"well%4:02:00::","call_ids":["senseval2.d002.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"well%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006627,"input_uncached_usd":0.000041899999999999995,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.323769582959358}
{"item_id":"senseval2.d002.s027.t000","gold_sense_keys":["remember%2:31:00::"],"candidates":[{"index":1,"sense_key":"remember%2:31:00::","synset_id":"remember.v.01"},{"index":2,"sense_key":"remember%2:31:01::","synset_id":"remember.v.02"},{"index":3,"sense_key":"remember%2:31:02::","synset_id":"remember.v.03"},{"index":4,"sense_key":"remember%2:40:00::","synset_id":"remember.v.04"},{"index":5,"sense_key":"remember%2:32:07::","synset_id":"remember.v.05"},{"index":6,"sense_key":"remember%2:32:00::","synset_id":"commend.v.05"},{"index":7,"sense_key":"remember%2:31:08::","synset_id":"remember.v.07"},{"index":8,"sense_key":"remember%2:31:09::","synset_id":"commemorate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"remember%2:31:02::","call_ids":["senseval2.d002.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"remember%2:31:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006264500000000001,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.906906874966808}
{"item_id":"senseval2.d002.s027.t001","gold_sense_keys":["truly%4:02:00::","truly%4:02:04::"],"candidates":[{"index":1,"sense_key":"truly%4:02:00::","synset_id":"truly.r.01"},{"index":2,"sense_key":"truly%4:02:01::","synset_id":"rightfully.r.01"},{"index":3,"sense_key":"truly%4:02:03::","synset_id":"sincerely.r.01"},{"index":4,"sense_key":"truly%4:02:04::","synset_id":"in_truth.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truly%4:02:00::","call_ids":["senseval2.d002.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":1426,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059345,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0005704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.791287875035778}
{"item_id":"senseval2.d002.s027.t003","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025315,"input_uncached_usd":0.000016349999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.886976749985479}
{"item_id":"senseval2.d002.s028.t000","gold_sense_keys":["education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036595,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.464057374978438}
{"item_id":"senseval2.d002.s028.t001","gold_sense_keys":["proceed%2:38:00::"],"candidates":[{"index":1,"sense_key":"proceed%2:32:00::","synset_id":"continue.v.02"},{"index":2,"sense_key":"proceed%2:38:00::","synset_id":"proceed.v.02"},{"index":3,"sense_key":"proceed%2:41:00::","synset_id":"go.v.02"},{"index":4,"sense_key":"proceed%2:30:00::","synset_id":"proceed.v.04"},{"index":5,"sense_key":"proceed%2:42:00::","synset_id":"continue.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"proceed%2:42:00::","call_ids":["senseval2.d002.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"proceed%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004927,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.708109208964743}
{"item_id":"senseval2.d002.s028.t003","gold_sense_keys":["competent%3:00:00::"],"candidates":[{"index":1,"sense_key":"competent%3:00:00::","synset_id":"competent.a.01"},{"index":2,"sense_key":"competent%5:00:00:adequate:00","synset_id":"competent.s.01"},{"index":3,"sense_key":"competent%3:00:02::","synset_id":"competent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"competent%5:00:00:adequate:00","call_ids":["senseval2.d002.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"competent%5:00:00:adequate:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":365,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033184999999999996,"input_uncached_usd":0.00001825,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.482388333010022}
{"item_id":"senseval2.d002.s029.t000","gold_sense_keys":["sense%1:10:00::"],"candidates":[{"index":1,"sense_key":"sense%1:09:05::","synset_id":"sense.n.01"},{"index":2,"sense_key":"sense%1:10:00::","synset_id":"sense.n.02"},{"index":3,"sense_key":"sense%1:09:02::","synset_id":"sense.n.03"},{"index":4,"sense_key":"sense%1:09:04::","synset_id":"common_sense.n.01"},{"index":5,"sense_key":"sense%1:09:06::","synset_id":"sense.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sense%1:10:00::","call_ids":["senseval2.d002.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sense%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018564999999999999,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0457512919674627}
{"item_id":"senseval2.d002.s029.t001","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":331,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017654999999999998,"input_uncached_usd":0.00001655,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.811534209002275}
{"item_id":"senseval2.d002.s029.t002","gold_sense_keys":["compare%2:31:01::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"compare%2:31:01::","call_ids":["senseval2.d002.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"compare%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002844,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.926809167023748}
{"item_id":"senseval2.d002.s029.t003","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s029.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035769999999999997,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.855714709032327}
{"item_id":"senseval2.d002.s030.t000","gold_sense_keys":["brilliant%5:00:00:intelligent:00"],"candidates":[{"index":1,"sense_key":"brilliant%5:00:00:superior:02","synset_id":"brilliant.s.01"},{"index":2,"sense_key":"brilliant%5:00:00:intelligent:00","synset_id":"brainy.s.01"},{"index":3,"sense_key":"brilliant%5:00:00:impressive:00","synset_id":"brilliant.s.03"},{"index":4,"sense_key":"brilliant%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":5,"sense_key":"brilliant%5:00:00:bright:00","synset_id":"brilliant.s.05"},{"index":6,"sense_key":"brilliant%5:00:00:reverberant:00","synset_id":"bright.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brilliant%5:00:00:intelligent:00","call_ids":["senseval2.d002.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brilliant%5:00:00:intelligent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002634,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.857191167015117}
{"item_id":"senseval2.d002.s030.t001","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d002.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033235,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.821175166987814}
{"item_id":"senseval2.d002.s031.t000","gold_sense_keys":["competent%3:00:00::"],"candidates":[{"index":1,"sense_key":"competent%3:00:00::","synset_id":"competent.a.01"},{"index":2,"sense_key":"competent%5:00:00:adequate:00","synset_id":"competent.s.01"},{"index":3,"sense_key":"competent%3:00:02::","synset_id":"competent.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"competent%3:00:00::","call_ids":["senseval2.d002.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"competent%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045894999999999996,"input_uncached_usd":0.000017349999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.718149167019874}
{"item_id":"senseval2.d002.s031.t002","gold_sense_keys":["serve%2:42:02::","serve%2:42:03::"],"candidates":[{"index":1,"sense_key":"serve%2:42:03::","synset_id":"serve.v.01"},{"index":2,"sense_key":"serve%2:33:00::","synset_id":"serve.v.02"},{"index":3,"sense_key":"serve%2:42:01::","synset_id":"serve.v.03"},{"index":4,"sense_key":"serve%2:41:00::","synset_id":"service.v.01"},{"index":5,"sense_key":"serve%2:34:01::","synset_id":"serve.v.05"},{"index":6,"sense_key":"serve%2:34:00::","synset_id":"serve.v.06"},{"index":7,"sense_key":"serve%2:41:02::","synset_id":"serve.v.07"},{"index":8,"sense_key":"serve%2:42:02::","synset_id":"serve.v.08"},{"index":9,"sense_key":"serve%2:42:00::","synset_id":"serve.v.09"},{"index":10,"sense_key":"serve%2:41:01::","synset_id":"serve.v.10"},{"index":11,"sense_key":"serve%2:35:00::","synset_id":"serve.v.11"},{"index":12,"sense_key":"serve%2:42:12::","synset_id":"suffice.v.01"},{"index":13,"sense_key":"serve%2:41:13::","synset_id":"serve.v.13"},{"index":14,"sense_key":"serve%2:35:01::","synset_id":"serve.v.14"},{"index":15,"sense_key":"serve%2:33:01::","synset_id":"serve.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"serve%2:42:02::","call_ids":["senseval2.d002.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"serve%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":920,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002316,"input_uncached_usd":0.000046,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3207853339845315}
{"item_id":"senseval2.d002.s031.t003","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"well%4:02:10::","call_ids":["senseval2.d002.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"well%4:02:10::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00076435,"input_uncached_usd":0.00004115,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.992299791018013}
{"item_id":"senseval2.d002.s032.t000","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":321,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025285,"input_uncached_usd":0.00001605,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3101321660215035}
{"item_id":"senseval2.d002.s032.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"important%3:00:04::","call_ids":["senseval2.d002.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"important%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033785,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.291795708006248}
{"item_id":"senseval2.d002.s032.t002","gold_sense_keys":["factor%1:09:00::","factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"factor%1:11:00::","call_ids":["senseval2.d002.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026504999999999996,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.365731584024616}
{"item_id":"senseval2.d002.s032.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":324,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004322,"input_uncached_usd":0.0000162,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.404418500023894}
{"item_id":"senseval2.d002.s032.t004","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["senseval2.d002.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.00089365,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.427651999983937}
{"item_id":"senseval2.d002.s033.t000","gold_sense_keys":["problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["senseval2.d002.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006916,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.256631542055402}
{"item_id":"senseval2.d002.s033.t001","gold_sense_keys":["equity%1:07:00::"],"candidates":[{"index":1,"sense_key":"equity%1:21:01::","synset_id":"equity.n.01"},{"index":2,"sense_key":"equity%1:21:00::","synset_id":"equity.n.02"},{"index":3,"sense_key":"equity%1:07:00::","synset_id":"fairness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"equity%1:07:00::","call_ids":["senseval2.d002.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"equity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012734999999999998,"input_uncached_usd":0.00001855,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2537249999586493}
{"item_id":"senseval2.d002.s033.t002","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020234999999999999,"input_uncached_usd":0.00001675,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5190594579908065}
{"item_id":"senseval2.d002.s033.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"problem%1:10:00::","call_ids":["senseval2.d002.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"problem%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":391,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043555,"input_uncached_usd":0.000019549999999999997,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0437513329670765}
{"item_id":"senseval2.d002.s034.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006376,"input_uncached_usd":0.0000168,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.018920208036434}
{"item_id":"senseval2.d002.s034.t003","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval2.d002.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027895,"input_uncached_usd":0.00004215,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.265153707994614}
{"item_id":"senseval2.d002.s034.t004","gold_sense_keys":["educate%2:41:00::"],"candidates":[{"index":1,"sense_key":"educate%2:41:00::","synset_id":"educate.v.01"},{"index":2,"sense_key":"educate%2:31:00::","synset_id":"train.v.01"},{"index":3,"sense_key":"educate%2:41:01::","synset_id":"educate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educate%2:41:00::","call_ids":["senseval2.d002.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educate%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025739999999999997,"input_uncached_usd":0.0000206,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.005314540991094}
{"item_id":"senseval2.d002.s034.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018245,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1380161669803783}
{"item_id":"senseval2.d002.s034.t006","gold_sense_keys":["better%4:02:00::"],"candidates":[{"index":1,"sense_key":"better%4:02:00::","synset_id":"better.r.01"},{"index":2,"sense_key":"better%4:02:01::","synset_id":"better.r.02"},{"index":3,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":4,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":5,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":6,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":7,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":8,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":9,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":10,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":11,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":12,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":13,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":14,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":15,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"better%4:02:00::","call_ids":["senseval2.d002.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"better%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004907,"input_uncached_usd":0.0000491,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.129468208004255}
{"item_id":"senseval2.d002.s034.t008","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"get%2:30:03::","call_ids":["senseval2.d002.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"get%2:30:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1908,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000281,"input_uncached_usd":0.0000954,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5914630420156755}
{"item_id":"senseval2.d002.s034.t010","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d002.s034.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023475,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.959873542014975}
{"item_id":"senseval2.d002.s034.t011","gold_sense_keys":["more%4:02:01::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:01::","call_ids":["senseval2.d002.s034.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045994999999999993,"input_uncached_usd":0.00001795,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.716719040996395}
{"item_id":"senseval2.d002.s035.t002","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":348,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043339999999999996,"input_uncached_usd":0.0000174,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.559876332990825}
{"item_id":"senseval2.d002.s035.t003","gold_sense_keys":["union%1:14:01::"],"candidates":[{"index":1,"sense_key":"union%1:14:01::","synset_id":"union.n.01"},{"index":2,"sense_key":"union%1:15:00::","synset_id":"union.n.02"},{"index":3,"sense_key":"union%1:04:02::","synset_id":"coupling.n.03"},{"index":4,"sense_key":"union%1:26:00::","synset_id":"union.n.04"},{"index":5,"sense_key":"union%1:26:02::","synset_id":"marriage.n.01"},{"index":6,"sense_key":"union%1:22:00::","synset_id":"union.n.06"},{"index":7,"sense_key":"union%1:14:02::","synset_id":"union.n.07"},{"index":8,"sense_key":"union%1:14:03::","synset_id":"union.n.08"},{"index":9,"sense_key":"union%1:11:00::","synset_id":"union.n.09"},{"index":10,"sense_key":"union%1:06:01::","synset_id":"union.n.10"},{"index":11,"sense_key":"union%1:04:00::","synset_id":"union.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"union%1:14:01::","call_ids":["senseval2.d002.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"union%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030385,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.815775291994214}
{"item_id":"senseval2.d002.s035.t004","gold_sense_keys":["mind%2:32:00::"],"candidates":[{"index":1,"sense_key":"mind%2:32:00::","synset_id":"mind.v.01"},{"index":2,"sense_key":"mind%2:31:02::","synset_id":"mind.v.02"},{"index":3,"sense_key":"mind%2:41:01::","synset_id":"take_care.v.02"},{"index":4,"sense_key":"mind%2:41:00::","synset_id":"heed.v.01"},{"index":5,"sense_key":"mind%2:31:03::","synset_id":"beware.v.01"},{"index":6,"sense_key":"mind%2:31:00::","synset_id":"mind.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mind%2:32:00::","call_ids":["senseval2.d002.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mind%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.000518,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.571292792039458}
{"item_id":"senseval2.d002.s035.t005","gold_sense_keys":["spread%2:32:00::","spread%2:35:00::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"spread%2:32:00::","call_ids":["senseval2.d002.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"spread%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029709999999999996,"input_uncached_usd":0.000034699999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2762321250047535}
{"item_id":"senseval2.d002.s035.t007","gold_sense_keys":["narrow%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"narrow%3:00:00::","synset_id":"narrow.a.01"},{"index":2,"sense_key":"narrow%5:00:00:limited:00","synset_id":"narrow.s.01"},{"index":3,"sense_key":"narrow%3:00:06::","synset_id":"narrow-minded.a.02"},{"index":4,"sense_key":"narrow%3:00:02::","synset_id":"narrow.a.04"},{"index":5,"sense_key":"narrow%5:00:00:careful:00","synset_id":"minute.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"narrow%5:00:00:limited:00","call_ids":["senseval2.d002.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"narrow%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026044999999999996,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.353917375032324}
{"item_id":"senseval2.d002.s035.t008","gold_sense_keys":["purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001828,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.13871129200561}
{"item_id":"senseval2.d002.s036.t003","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033289999999999996,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.088299916998949}
{"item_id":"senseval2.d002.s036.t004","gold_sense_keys":["find%2:31:10::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"find%2:31:10::","call_ids":["senseval2.d002.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"find%2:31:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1086,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004191,"input_uncached_usd":0.0000543,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.620200915960595}
{"item_id":"senseval2.d002.s036.t006","gold_sense_keys":["help%2:41:00::"],"candidates":[{"index":1,"sense_key":"help%2:41:00::","synset_id":"help.v.01"},{"index":2,"sense_key":"help%2:29:00::","synset_id":"help.v.02"},{"index":3,"sense_key":"help%2:42:00::","synset_id":"help.v.03"},{"index":4,"sense_key":"help%2:42:08::","synset_id":"help_oneself.v.01"},{"index":5,"sense_key":"help%2:34:00::","synset_id":"serve.v.05"},{"index":6,"sense_key":"help%2:41:12::","synset_id":"help.v.06"},{"index":7,"sense_key":"help%2:34:04::","synset_id":"avail.v.03"},{"index":8,"sense_key":"help%2:30:13::","synset_id":"help.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"help%2:42:00::","call_ids":["senseval2.d002.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"help%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021674999999999998,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.709706584049854}
{"item_id":"senseval2.d002.s036.t007","gold_sense_keys":["strike%2:38:08::","strike%2:41:02::"],"candidates":[{"index":1,"sense_key":"strike%2:35:00::","synset_id":"strike.v.01"},{"index":2,"sense_key":"strike%2:37:00::","synset_id":"affect.v.05"},{"index":3,"sense_key":"strike%2:35:01::","synset_id":"hit.v.02"},{"index":4,"sense_key":"strike%2:33:00::","synset_id":"strike.v.04"},{"index":5,"sense_key":"strike%2:32:00::","synset_id":"strike.v.05"},{"index":6,"sense_key":"strike%2:35:02::","synset_id":"hit.v.05"},{"index":7,"sense_key":"strike%2:41:00::","synset_id":"strike.v.07"},{"index":8,"sense_key":"strike%2:42:00::","synset_id":"fall.v.08"},{"index":9,"sense_key":"strike%2:41:02::","synset_id":"come_to.v.03"},{"index":10,"sense_key":"strike%2:35:03::","synset_id":"strike.v.10"},{"index":11,"sense_key":"strike%2:30:00::","synset_id":"strike.v.11"},{"index":12,"sense_key":"strike%2:40:00::","synset_id":"fall_upon.v.01"},{"index":13,"sense_key":"strike%2:36:00::","synset_id":"strike.v.13"},{"index":14,"sense_key":"strike%2:35:08::","synset_id":"strike.v.14"},{"index":15,"sense_key":"strike%2:39:00::","synset_id":"hit.v.09"},{"index":16,"sense_key":"strike%2:38:02::","synset_id":"hit.v.12"},{"index":17,"sense_key":"strike%2:38:08::","synset_id":"assume.v.05"},{"index":18,"sense_key":"strike%2:36:02::","synset_id":"mint.v.01"},{"index":19,"sense_key":"strike%2:35:10::","synset_id":"strickle.v.02"},{"index":20,"sense_key":"strike%2:35:09::","synset_id":"strike.v.20"},{"index":21,"sense_key":"strike%2:31:00::","synset_id":"strike.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":17,"chosen_sense_key":"strike%2:38:08::","call_ids":["senseval2.d002.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":17,"predicted_sense_key":"strike%2:38:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1257,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00029965,"input_uncached_usd":0.00006285,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.817602208000608}
{"item_id":"senseval2.d002.s036.t008","gold_sense_keys":["friendly%5:00:00:amicable:00"],"candidates":[{"index":1,"sense_key":"friendly%3:00:01::","synset_id":"friendly.a.01"},{"index":2,"sense_key":"friendly%5:00:00:amicable:00","synset_id":"friendly.s.01"},{"index":3,"sense_key":"friendly%3:00:03::","synset_id":"friendly.a.03"},{"index":4,"sense_key":"friendly%3:00:02::","synset_id":"friendly.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"friendly%5:00:00:amicable:00","call_ids":["senseval2.d002.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"friendly%5:00:00:amicable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020885,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.968591125041712}
{"item_id":"senseval2.d002.s036.t009","gold_sense_keys":["posture%1:09:00::"],"candidates":[{"index":1,"sense_key":"posture%1:07:00::","synset_id":"position.n.04"},{"index":2,"sense_key":"posture%1:07:01::","synset_id":"carriage.n.03"},{"index":3,"sense_key":"posture%1:09:00::","synset_id":"position.n.12"},{"index":4,"sense_key":"posture%1:07:02::","synset_id":"military_capability.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"posture%1:09:00::","call_ids":["senseval2.d002.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"posture%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013225,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.006555667030625}
{"item_id":"senseval2.d002.s036.t010","gold_sense_keys":["behalf%1:04:00::"],"candidates":[{"index":1,"sense_key":"behalf%1:04:00::","synset_id":"behalf.n.01"},{"index":2,"sense_key":"behalf%1:07:00::","synset_id":"behalf.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behalf%1:04:00::","call_ids":["senseval2.d002.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behalf%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005124000000000001,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.51897008297965}
{"item_id":"senseval2.d002.s036.t011","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"important%5:00:00:influential:00","call_ids":["senseval2.d002.s036.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"important%5:00:00:influential:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054435,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.17575441597728}
{"item_id":"senseval2.d002.s037.t000","gold_sense_keys":["shred%1:23:00::"],"candidates":[{"index":1,"sense_key":"shred%1:23:00::","synset_id":"shred.n.01"},{"index":2,"sense_key":"shred%1:06:00::","synset_id":"rag.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shred%1:23:00::","call_ids":["senseval2.d002.s037.t000__v1__a1","senseval2.d002.s037.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shred%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":736,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00033475,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.0002944,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.159763082978316}
{"item_id":"senseval2.d002.s037.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d002.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028345,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.398524290998466}
{"item_id":"senseval2.d002.s037.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026044999999999996,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3514880840084516}
{"item_id":"senseval2.d002.s037.t003","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"thing%1:09:02::","call_ids":["senseval2.d002.s037.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"thing%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002716,"input_uncached_usd":0.0000348,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.336849124985747}
{"item_id":"senseval2.d002.s037.t004","gold_sense_keys":["equal%3:00:00::"],"candidates":[{"index":1,"sense_key":"equal%3:00:00::","synset_id":"equal.a.01"},{"index":2,"sense_key":"equal%3:00:03::","synset_id":"adequate.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equal%3:00:00::","call_ids":["senseval2.d002.s037.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005120000000000001,"input_uncached_usd":0.0000192,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.519802124996204}
{"item_id":"senseval2.d002.s037.t006","gold_sense_keys":["differential%1:07:00::"],"candidates":[{"index":1,"sense_key":"differential%1:09:00::","synset_id":"derived_function.n.01"},{"index":2,"sense_key":"differential%1:07:00::","synset_id":"differential.n.02"},{"index":3,"sense_key":"differential%1:06:00::","synset_id":"differential_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"differential%1:07:00::","call_ids":["senseval2.d002.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"differential%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018145,"input_uncached_usd":0.00002145,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.182108166976832}
{"item_id":"senseval2.d002.s037.t007","gold_sense_keys":["result%2:42:00::"],"candidates":[{"index":1,"sense_key":"result%2:42:00::","synset_id":"result.v.01"},{"index":2,"sense_key":"result%2:42:02::","synset_id":"leave.v.07"},{"index":3,"sense_key":"result%2:30:00::","synset_id":"result.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%2:30:00::","call_ids":["senseval2.d002.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018109999999999998,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.413784708012827}
{"item_id":"senseval2.d002.s037.t008","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015255,"input_uncached_usd":0.00001815,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.883920708030928}
{"item_id":"senseval2.d002.s037.t009","gold_sense_keys":["differential%1:07:00::"],"candidates":[{"index":1,"sense_key":"differential%1:09:00::","synset_id":"derived_function.n.01"},{"index":2,"sense_key":"differential%1:07:00::","synset_id":"differential.n.02"},{"index":3,"sense_key":"differential%1:06:00::","synset_id":"differential_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"differential%1:07:00::","call_ids":["senseval2.d002.s037.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"differential%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002586,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.505757374980021}
{"item_id":"senseval2.d002.s038.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval2.d002.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007706,"input_uncached_usd":0.000021799999999999998,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.039179917017464}
{"item_id":"senseval2.d002.s038.t002","gold_sense_keys":["sure%3:00:00::"],"candidates":[{"index":1,"sense_key":"sure%3:00:00::","synset_id":"certain.a.02"},{"index":2,"sense_key":"sure%5:00:00:careful:00","synset_id":"certain.s.03"},{"index":3,"sense_key":"sure%3:00:04::","synset_id":"certain.a.04"},{"index":4,"sense_key":"sure%5:00:00:secure:02","synset_id":"sure.s.02"},{"index":5,"sense_key":"sure%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":6,"sense_key":"sure%5:00:00:trustworthy:00","synset_id":"sure.s.04"},{"index":7,"sense_key":"sure%5:00:00:true:00","synset_id":"sure.s.05"},{"index":8,"sense_key":"sure%5:00:00:steady:00","synset_id":"sure.s.06"},{"index":9,"sense_key":"sure%5:00:02:certain:01","synset_id":"indisputable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sure%3:00:00::","call_ids":["senseval2.d002.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sure%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022064999999999997,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5791480829939246}
{"item_id":"senseval2.d002.s039.t000","gold_sense_keys":["wish%2:37:02::"],"candidates":[{"index":1,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":2,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":3,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"},{"index":4,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":5,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":6,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wish%2:37:02::","call_ids":["senseval2.d002.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wish%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036619999999999996,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.799870749993715}
{"item_id":"senseval2.d002.s039.t001","gold_sense_keys":["serious%3:00:00::"],"candidates":[{"index":1,"sense_key":"serious%3:00:00::","synset_id":"serious.a.01"},{"index":2,"sense_key":"serious%5:00:00:important:00","synset_id":"serious.s.01"},{"index":3,"sense_key":"serious%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":4,"sense_key":"serious%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":5,"sense_key":"serious%3:00:02::","synset_id":"unplayful.a.01"},{"index":6,"sense_key":"serious%5:00:00:difficult:00","synset_id":"serious.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"serious%3:00:00::","call_ids":["senseval2.d002.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"serious%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006246499999999999,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.289873166999314}
{"item_id":"senseval2.d002.s039.t002","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004108,"input_uncached_usd":0.000020399999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.8215045419638045}
{"item_id":"senseval2.d002.s039.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"education%1:04:00::","call_ids":["senseval2.d002.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"education%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034314999999999997,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.959890290978365}
{"item_id":"senseval2.d002.s039.t004","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030439999999999997,"input_uncached_usd":0.000042,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3558370000100695}
{"item_id":"senseval2.d002.s039.t005","gold_sense_keys":["exactly%4:02:01::"],"candidates":[{"index":1,"sense_key":"exactly%4:02:01::","synset_id":"precisely.r.01"},{"index":2,"sense_key":"exactly%4:02:04::","synset_id":"precisely.r.03"},{"index":3,"sense_key":"exactly%4:02:02::","synset_id":"precisely.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exactly%4:02:01::","call_ids":["senseval2.d002.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exactly%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023519999999999997,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.769319666957017}
{"item_id":"senseval2.d002.s039.t006","gold_sense_keys":["just%4:02:05::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"just%4:02:05::","call_ids":["senseval2.d002.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"just%4:02:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":2130,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.0008826999999999999,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.000852,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.443452917039394}
{"item_id":"senseval2.d002.s039.t007","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"important%3:00:02::","call_ids":["senseval2.d002.s039.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"important%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00085305,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.248212333011907}
{"item_id":"senseval2.d002.s040.t002","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026754999999999997,"input_uncached_usd":0.000030749999999999995,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.649233000003733}
{"item_id":"senseval2.d002.s040.t003","gold_sense_keys":["scattered%5:00:00:distributed:00"],"candidates":[{"index":1,"sense_key":"scattered%5:00:00:distributed:00","synset_id":"scattered.s.01"},{"index":2,"sense_key":"scattered%5:00:00:incoherent:00","synset_id":"confused.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scattered%5:00:00:distributed:00","call_ids":["senseval2.d002.s040.t003__v1__a1","senseval2.d002.s040.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scattered%5:00:00:distributed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":895,"cached_input_tokens":0,"output_tokens":800,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036474999999999995,"input_uncached_usd":0.000044750000000000004,"input_cached_usd":0.0,"output_usd":0.00031999999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.330780209042132}
{"item_id":"senseval2.d002.s040.t005","gold_sense_keys":["poor%3:00:00::","poor%3:00:03::"],"candidates":[{"index":1,"sense_key":"poor%5:00:00:unfortunate:00","synset_id":"hapless.s.01"},{"index":2,"sense_key":"poor%3:00:00::","synset_id":"poor.a.02"},{"index":3,"sense_key":"poor%3:00:03::","synset_id":"poor.a.03"},{"index":4,"sense_key":"poor%3:00:02::","synset_id":"poor.a.04"},{"index":5,"sense_key":"poor%5:00:00:insufficient:00","synset_id":"inadequate.s.01"},{"index":6,"sense_key":"poor%5:00:00:bad:00","synset_id":"poor.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"poor%3:00:03::","call_ids":["senseval2.d002.s040.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"poor%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003941,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.483944125007838}
{"item_id":"senseval2.d002.s040.t006","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval2.d002.s040.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00095205,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.635083083005156}
{"item_id":"senseval2.d002.s041.t001","gold_sense_keys":["majority%1:07:00::"],"candidates":[{"index":1,"sense_key":"majority%1:07:00::","synset_id":"majority.n.01"},{"index":2,"sense_key":"majority%1:23:00::","synset_id":"majority.n.02"},{"index":3,"sense_key":"majority%1:28:00::","synset_id":"majority.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"majority%1:07:00::","call_ids":["senseval2.d002.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"majority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013115,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7622689160052687}
{"item_id":"senseval2.d002.s041.t003","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024189999999999997,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.640626207983587}
{"item_id":"senseval2.d002.s041.t004","gold_sense_keys":["know%2:31:01::","know%2:31:02::","know%2:31:12::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040649999999999996,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.306906208978035}
{"item_id":"senseval2.d002.s041.t005","gold_sense_keys":["effort%1:04:00::"],"candidates":[{"index":1,"sense_key":"effort%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"effort%1:04:01::","synset_id":"effort.n.02"},{"index":3,"sense_key":"effort%1:04:02::","synset_id":"feat.n.01"},{"index":4,"sense_key":"effort%1:04:03::","synset_id":"campaign.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"effort%1:04:03::","call_ids":["senseval2.d002.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"effort%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007247999999999999,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.635662665998098}
{"item_id":"senseval2.d002.s041.t006","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030715,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.028688124963082}
{"item_id":"senseval2.d002.s041.t007","gold_sense_keys":["reform%1:04:00::"],"candidates":[{"index":1,"sense_key":"reform%1:04:00::","synset_id":"reform.n.01"},{"index":2,"sense_key":"reform%1:04:01::","synset_id":"reform.n.02"},{"index":3,"sense_key":"reform%1:04:02::","synset_id":"reform.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reform%1:04:01::","call_ids":["senseval2.d002.s041.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reform%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020764999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1829840000136755}
{"item_id":"senseval2.d002.s041.t008","gold_sense_keys":["doomed%5:00:00:unfortunate:00"],"candidates":[{"index":1,"sense_key":"doomed%5:00:00:dead:01","synset_id":"doomed.s.01"},{"index":2,"sense_key":"doomed%5:00:00:lost:02","synset_id":"cursed.s.01"},{"index":3,"sense_key":"doomed%5:00:00:unfortunate:00","synset_id":"doomed.s.03"},{"index":4,"sense_key":"doomed%5:00:00:certain:03","synset_id":"doomed.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"doomed%5:00:00:certain:03","call_ids":["senseval2.d002.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"doomed%5:00:00:certain:03","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023779999999999998,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.522044708020985}
{"item_id":"senseval2.d002.s042.t000","gold_sense_keys":["really%4:02:02::"],"candidates":[{"index":1,"sense_key":"really%4:02:03::","synset_id":"truly.r.01"},{"index":2,"sense_key":"really%4:02:04::","synset_id":"actually.r.01"},{"index":3,"sense_key":"really%4:02:02::","synset_id":"in_truth.r.01"},{"index":4,"sense_key":"really%4:02:00::","synset_id":"very.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"really%4:02:00::","call_ids":["senseval2.d002.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"really%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.000954,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.506239333015401}
{"item_id":"senseval2.d002.s042.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s042.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035519999999999996,"input_uncached_usd":0.000041599999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.212197542015929}
{"item_id":"senseval2.d002.s042.t002","gold_sense_keys":["need%2:42:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d002.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"need%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":2769,"reasoning_output_tokens":2752},"cost":{"currency":"USD","total_usd":0.00112975,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0011076,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.702158125001006}
{"item_id":"senseval2.d002.s042.t003","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":1042,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045835,"input_uncached_usd":0.00004155,"input_cached_usd":0.0,"output_usd":0.0004168,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.563347125018481}
{"item_id":"senseval2.d002.s042.t005","gold_sense_keys":["assimilate%2:31:00::"],"candidates":[{"index":1,"sense_key":"assimilate%2:31:00::","synset_id":"absorb.v.02"},{"index":2,"sense_key":"assimilate%2:30:00::","synset_id":"assimilate.v.02"},{"index":3,"sense_key":"assimilate%2:30:01::","synset_id":"assimilate.v.03"},{"index":4,"sense_key":"assimilate%2:35:00::","synset_id":"assimilate.v.04"},{"index":5,"sense_key":"assimilate%2:30:02::","synset_id":"assimilate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assimilate%2:31:00::","call_ids":["senseval2.d002.s042.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assimilate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006459500000000001,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.578114166972227}
{"item_id":"senseval2.d002.s043.t000","gold_sense_keys":["respect%1:09:00::"],"candidates":[{"index":1,"sense_key":"respect%1:09:00::","synset_id":"respect.n.01"},{"index":2,"sense_key":"respect%1:26:00::","synset_id":"esteem.n.01"},{"index":3,"sense_key":"respect%1:09:01::","synset_id":"respect.n.03"},{"index":4,"sense_key":"respect%1:04:03::","synset_id":"deference.n.01"},{"index":5,"sense_key":"respect%1:04:02::","synset_id":"obedience.n.03"},{"index":6,"sense_key":"respect%1:12:00::","synset_id":"regard.n.06"},{"index":7,"sense_key":"respect%1:07:00::","synset_id":"deference.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respect%1:09:00::","call_ids":["senseval2.d002.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respect%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024334999999999998,"input_uncached_usd":0.000032149999999999995,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.726438916986808}
{"item_id":"senseval2.d002.s043.t005","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"},{"index":2,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":3,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":4,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s043.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013399999999999998,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2717482909793034}
{"item_id":"senseval2.d002.s043.t007","gold_sense_keys":["concerned%3:00:00::","concerned%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":2,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concerned%3:00:00::","call_ids":["senseval2.d002.s043.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concerned%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":1234,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.000517,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.0004936,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.063208708015736}
{"item_id":"senseval2.d002.s044.t001","gold_sense_keys":["inclined%3:00:02::"],"candidates":[{"index":1,"sense_key":"inclined%3:00:02::","synset_id":"inclined.a.01"},{"index":2,"sense_key":"inclined%3:00:01::","synset_id":"inclined.a.02"},{"index":3,"sense_key":"inclined%5:00:00:willing:00","synset_id":"disposed.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inclined%3:00:02::","call_ids":["senseval2.d002.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inclined%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025949999999999997,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.271680541976821}
{"item_id":"senseval2.d002.s044.t002","gold_sense_keys":["echo%2:32:00::"],"candidates":[{"index":1,"sense_key":"echo%2:32:00::","synset_id":"repeat.v.04"},{"index":2,"sense_key":"echo%2:39:00::","synset_id":"resound.v.01"},{"index":3,"sense_key":"echo%2:42:00::","synset_id":"echo.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"echo%2:32:00::","call_ids":["senseval2.d002.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"echo%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015664999999999999,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7718904999783263}
{"item_id":"senseval2.d002.s044.t003","gold_sense_keys":["conventional%3:00:00::","conventional%3:00:01::","conventional%5:00:00:orthodox:00"],"candidates":[{"index":1,"sense_key":"conventional%3:00:00::","synset_id":"conventional.a.01"},{"index":2,"sense_key":"conventional%5:00:00:orthodox:00","synset_id":"conventional.s.01"},{"index":3,"sense_key":"conventional%3:00:02::","synset_id":"conventional.a.03"},{"index":4,"sense_key":"conventional%3:00:01::","synset_id":"conventional.a.04"},{"index":5,"sense_key":"conventional%5:00:00:nonrepresentational:00","synset_id":"conventional.s.02"},{"index":6,"sense_key":"conventional%5:00:00:traditional:00","synset_id":"conventional.s.03"},{"index":7,"sense_key":"conventional%5:00:00:formal:01","synset_id":"ceremonious.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conventional%5:00:00:orthodox:00","call_ids":["senseval2.d002.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conventional%5:00:00:orthodox:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006774,"input_uncached_usd":0.000031,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.834108125010971}
{"item_id":"senseval2.d002.s044.t004","gold_sense_keys":["pap%1:10:00::"],"candidates":[{"index":1,"sense_key":"pap%1:10:00::","synset_id":"pap.n.01"},{"index":2,"sense_key":"pap%1:13:00::","synset_id":"soft_diet.n.01"},{"index":3,"sense_key":"pap%1:08:00::","synset_id":"nipple.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pap%1:10:00::","call_ids":["senseval2.d002.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pap%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018224999999999998,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2938323749694973}
{"item_id":"senseval2.d002.s045.t000","gold_sense_keys":["thus%4:02:00::"],"candidates":[{"index":1,"sense_key":"thus%4:02:00::","synset_id":"therefore.r.01"},{"index":2,"sense_key":"thus%4:02:01::","synset_id":"thus.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thus%4:02:00::","call_ids":["senseval2.d002.s045.t000__v1__a1","senseval2.d002.s045.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thus%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":736,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00034044999999999995,"input_uncached_usd":0.000046049999999999994,"input_cached_usd":0.0,"output_usd":0.0002944,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.936448792053852}
{"item_id":"senseval2.d002.s045.t001","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval2.d002.s045.t001__v1__a1","senseval2.d002.s045.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":1185,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005161499999999999,"input_uncached_usd":0.000042149999999999995,"input_cached_usd":0.0,"output_usd":0.0004739999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.375259667052887}
{"item_id":"senseval2.d002.s045.t002","gold_sense_keys":["governor%1:18:00::"],"candidates":[{"index":1,"sense_key":"governor%1:18:00::","synset_id":"governor.n.01"},{"index":2,"sense_key":"governor%1:06:00::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"governor%1:18:00::","call_ids":["senseval2.d002.s045.t002__v1__a1","senseval2.d002.s045.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"governor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":1952,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00082215,"input_uncached_usd":0.00004135,"input_cached_usd":0.0,"output_usd":0.0007807999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.594263459031936}
{"item_id":"senseval2.d002.s045.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["senseval2.d002.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003348,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.638988499995321}
{"item_id":"senseval2.d002.s045.t004","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s045.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.000855,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.819524792023003}
{"item_id":"senseval2.d002.s045.t005","gold_sense_keys":["announce%2:32:02::"],"candidates":[{"index":1,"sense_key":"announce%2:32:00::","synset_id":"announce.v.01"},{"index":2,"sense_key":"announce%2:32:02::","synset_id":"announce.v.02"},{"index":3,"sense_key":"announce%2:32:07::","synset_id":"announce.v.03"},{"index":4,"sense_key":"announce%2:32:01::","synset_id":"announce.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"announce%2:32:02::","call_ids":["senseval2.d002.s045.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"announce%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003879,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.600283833977301}
{"item_id":"senseval2.d002.s045.t006","gold_sense_keys":["country%1:14:00::","country%1:14:01::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["senseval2.d002.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":850,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036855,"input_uncached_usd":0.00002855,"input_cached_usd":0.0,"output_usd":0.00033999999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.216115417017136}
{"item_id":"senseval2.d002.s045.t007","gold_sense_keys":["need%2:34:00::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"need%2:42:00::","call_ids":["senseval2.d002.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"need%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004384,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.885226290964056}
{"item_id":"senseval2.d002.s045.t009","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval2.d002.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003106,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.266959916043561}
{"item_id":"senseval2.d002.s045.t010","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval2.d002.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003896,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.569381999957841}
{"item_id":"senseval2.d002.s046.t001","gold_sense_keys":["development%1:04:01::","development%1:22:02::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["senseval2.d002.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":792,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007372,"input_uncached_usd":0.0000396,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.795084582990967}
{"item_id":"senseval2.d002.s046.t002","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval2.d002.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031025,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.131502500036731}
{"item_id":"senseval2.d002.s046.t003","gold_sense_keys":["governor%1:18:00::"],"candidates":[{"index":1,"sense_key":"governor%1:18:00::","synset_id":"governor.n.01"},{"index":2,"sense_key":"governor%1:06:00::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"governor%1:18:00::","call_ids":["senseval2.d002.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"governor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":1490,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006152,"input_uncached_usd":0.0000192,"input_cached_usd":0.0,"output_usd":0.000596,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.089212124992628}
{"item_id":"senseval2.d002.s046.t004","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval2.d002.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026175,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.547044584003743}
{"item_id":"senseval2.d002.s046.t005","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:01::","call_ids":["senseval2.d002.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00082,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.78259508300107}
{"item_id":"senseval2.d002.s046.t007","gold_sense_keys":["need%1:17:00::","need%1:26:00::"],"candidates":[{"index":1,"sense_key":"need%1:26:00::","synset_id":"need.n.01"},{"index":2,"sense_key":"need%1:17:00::","synset_id":"need.n.02"},{"index":3,"sense_key":"need%1:03:00::","synset_id":"motivation.n.01"},{"index":4,"sense_key":"need%1:26:01::","synset_id":"indigence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"need%1:17:00::","call_ids":["senseval2.d002.s046.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"need%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044315,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.083838041988201}
{"item_id":"senseval2.d002.s047.t002","gold_sense_keys":["extend%2:30:01::"],"candidates":[{"index":1,"sense_key":"extend%2:30:01::","synset_id":"widen.v.04"},{"index":2,"sense_key":"extend%2:42:03::","synset_id":"run.v.03"},{"index":3,"sense_key":"extend%2:42:01::","synset_id":"cover.v.03"},{"index":4,"sense_key":"extend%2:40:05::","synset_id":"extend.v.04"},{"index":5,"sense_key":"extend%2:29:01::","synset_id":"exsert.v.01"},{"index":6,"sense_key":"extend%2:42:02::","synset_id":"extend.v.06"},{"index":7,"sense_key":"extend%2:40:04::","synset_id":"offer.v.05"},{"index":8,"sense_key":"extend%2:29:00::","synset_id":"stretch.v.02"},{"index":9,"sense_key":"extend%2:30:09::","synset_id":"extend.v.09"},{"index":10,"sense_key":"extend%2:30:02::","synset_id":"prolong.v.01"},{"index":11,"sense_key":"extend%2:38:10::","synset_id":"unfold.v.03"},{"index":12,"sense_key":"extend%2:38:14::","synset_id":"gallop.v.03"},{"index":13,"sense_key":"extend%2:35:10::","synset_id":"extend.v.13"},{"index":14,"sense_key":"extend%2:34:00::","synset_id":"strain.v.03"},{"index":15,"sense_key":"extend%2:30:15::","synset_id":"extend.v.15"},{"index":16,"sense_key":"extend%2:30:06::","synset_id":"carry.v.09"},{"index":17,"sense_key":"extend%2:30:12::","synset_id":"extend.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extend%2:30:01::","call_ids":["senseval2.d002.s047.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extend%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1087,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036794999999999997,"input_uncached_usd":0.00005435,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.744000208971556}
{"item_id":"senseval2.d002.s047.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":1362,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056345,"input_uncached_usd":0.00001865,"input_cached_usd":0.0,"output_usd":0.0005448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.39549858297687}
{"item_id":"senseval2.d002.s047.t004","gold_sense_keys":["system%1:14:00::"],"candidates":[{"index":1,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":2,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":3,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":4,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":5,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":7,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":8,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":9,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"system%1:09:02::","call_ids":["senseval2.d002.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"system%1:09:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004029,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.889424290973693}
{"item_id":"senseval2.d002.s048.t000","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006664999999999999,"input_uncached_usd":0.000019699999999999998,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.826961250044405}
{"item_id":"senseval2.d002.s048.t002","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d002.s048.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029749999999999997,"input_uncached_usd":0.0000351,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.79689237504499}
{"item_id":"senseval2.d002.s048.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043474999999999997,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.32645291701192}
{"item_id":"senseval2.d002.s048.t004","gold_sense_keys":["program%1:09:01::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"program%1:09:01::","call_ids":["senseval2.d002.s048.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"program%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004255,"input_uncached_usd":0.0000351,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.874252875044476}
{"item_id":"senseval2.d002.s049.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058575,"input_uncached_usd":0.00004175,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.641186500026379}
{"item_id":"senseval2.d002.s049.t001","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["senseval2.d002.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043684999999999997,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0086558340117335}
{"item_id":"senseval2.d002.s049.t002","gold_sense_keys":["early%3:00:00::","early%3:00:02::"],"candidates":[{"index":1,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":2,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":3,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":5,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":6,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"early%5:00:00:young:00","call_ids":["senseval2.d002.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"early%5:00:00:young:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003172,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.298174915951677}
{"item_id":"senseval2.d002.s049.t003","gold_sense_keys":["exposure%1:04:06::"],"candidates":[{"index":1,"sense_key":"exposure%1:07:00::","synset_id":"exposure.n.01"},{"index":2,"sense_key":"exposure%1:04:06::","synset_id":"exposure.n.02"},{"index":3,"sense_key":"exposure%1:10:00::","synset_id":"exposure.n.03"},{"index":4,"sense_key":"exposure%1:09:00::","synset_id":"exposure.n.04"},{"index":5,"sense_key":"exposure%1:26:01::","synset_id":"vulnerability.n.01"},{"index":6,"sense_key":"exposure%1:23:00::","synset_id":"exposure.n.06"},{"index":7,"sense_key":"exposure%1:06:00::","synset_id":"photograph.n.01"},{"index":8,"sense_key":"exposure%1:04:02::","synset_id":"exposure.n.08"},{"index":9,"sense_key":"exposure%1:04:05::","synset_id":"exposure.n.09"},{"index":10,"sense_key":"exposure%1:04:01::","synset_id":"exposure.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exposure%1:04:06::","call_ids":["senseval2.d002.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exposure%1:04:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034905,"input_uncached_usd":0.00003545,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.276904416969046}
{"item_id":"senseval2.d002.s049.t004","gold_sense_keys":["schooling%1:09:00::"],"candidates":[{"index":1,"sense_key":"schooling%1:04:00::","synset_id":"schooling.n.01"},{"index":2,"sense_key":"schooling%1:09:00::","synset_id":"school.n.03"},{"index":3,"sense_key":"schooling%1:04:01::","synset_id":"schooling.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"schooling%1:09:00::","call_ids":["senseval2.d002.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"schooling%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023219999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.721037749957759}
{"item_id":"senseval2.d002.s049.t005","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["senseval2.d002.s049.t005__v1__a1","senseval2.d002.s049.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":861,"cached_input_tokens":0,"output_tokens":1824,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00077265,"input_uncached_usd":0.00004305,"input_cached_usd":0.0,"output_usd":0.0007296,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.222396917000879}
{"item_id":"senseval2.d002.s049.t006","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s049.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026534999999999997,"input_uncached_usd":0.00002855,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.677374999970198}
{"item_id":"senseval2.d002.s049.t007","gold_sense_keys":["first%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"first%3:00:00::","synset_id":"first.a.01"},{"index":2,"sense_key":"first%5:00:00:ordinal:00","synset_id":"first.s.01"},{"index":3,"sense_key":"first%5:00:02:opening:00","synset_id":"inaugural.s.01"},{"index":4,"sense_key":"first%5:00:01:opening:00","synset_id":"beginning.s.01"},{"index":5,"sense_key":"first%5:00:00:best:00","synset_id":"first.s.04"},{"index":6,"sense_key":"first%3:00:03::","synset_id":"first.a.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"first%5:00:00:ordinal:00","call_ids":["senseval2.d002.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"first%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018894999999999998,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.657154874992557}
{"item_id":"senseval2.d002.s049.t008","gold_sense_keys":["grade%1:14:00::"],"candidates":[{"index":1,"sense_key":"grade%1:14:00::","synset_id":"class.n.02"},{"index":2,"sense_key":"grade%1:26:00::","synset_id":"grade.n.02"},{"index":3,"sense_key":"grade%1:07:01::","synset_id":"grade.n.03"},{"index":4,"sense_key":"grade%1:23:00::","synset_id":"grad.n.01"},{"index":5,"sense_key":"grade%1:10:00::","synset_id":"grade.n.05"},{"index":6,"sense_key":"grade%1:09:00::","synset_id":"mark.n.01"},{"index":7,"sense_key":"grade%1:07:02::","synset_id":"grade.n.07"},{"index":8,"sense_key":"grade%1:07:00::","synset_id":"degree.n.01"},{"index":9,"sense_key":"grade%1:05:00::","synset_id":"grade.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"grade%1:07:00::","call_ids":["senseval2.d002.s049.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"grade%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005769,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.024880458018743}
{"item_id":"senseval2.d002.s049.t010","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"difference%1:11:00::","call_ids":["senseval2.d002.s049.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"difference%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036564999999999997,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.116679625003599}
{"item_id":"senseval2.d002.s049.t011","gold_sense_keys":["quickly%4:02:02::"],"candidates":[{"index":1,"sense_key":"quickly%4:02:00::","synset_id":"quickly.r.01"},{"index":2,"sense_key":"quickly%4:02:02::","synset_id":"promptly.r.01"},{"index":3,"sense_key":"quickly%4:02:03::","synset_id":"cursorily.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"quickly%4:02:02::","call_ids":["senseval2.d002.s049.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"quickly%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018209999999999998,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4472340419888496}
{"item_id":"senseval2.d002.s049.t013","gold_sense_keys":["wash_away%2:30:01::"],"candidates":[{"index":1,"sense_key":"wash_away%2:30:01::","synset_id":"wash_away.v.01"},{"index":2,"sense_key":"wash_away%2:30:00::","synset_id":"wash.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wash_away%2:30:01::","call_ids":["senseval2.d002.s049.t013__v1__a1","senseval2.d002.s049.t013__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wash_away%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":873,"cached_input_tokens":0,"output_tokens":800,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036365,"input_uncached_usd":0.000043650000000000004,"input_cached_usd":0.0,"output_usd":0.00031999999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.571543541038409}
{"item_id":"senseval2.d002.s050.t000","gold_sense_keys":["sum_up%2:32:00::"],"candidates":[{"index":1,"sense_key":"sum_up%2:32:00::","synset_id":"sum_up.v.01"},{"index":2,"sense_key":"sum_up%2:42:00::","synset_id":"summarize.v.02"},{"index":3,"sense_key":"sum_up%2:32:03::","synset_id":"total.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sum_up%2:32:00::","call_ids":["senseval2.d002.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sum_up%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025969999999999997,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.249215874995571}
{"item_id":"senseval2.d002.s050.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval2.d002.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058535,"input_uncached_usd":0.000041349999999999995,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.824879333027638}
{"item_id":"senseval2.d002.s050.t002","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005985500000000001,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.863982959010173}
{"item_id":"senseval2.d002.s050.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004705499999999999,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.03775191603927}
{"item_id":"senseval2.d002.s050.t004","gold_sense_keys":["reform%1:04:00::"],"candidates":[{"index":1,"sense_key":"reform%1:04:00::","synset_id":"reform.n.01"},{"index":2,"sense_key":"reform%1:04:01::","synset_id":"reform.n.02"},{"index":3,"sense_key":"reform%1:04:02::","synset_id":"reform.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reform%1:04:00::","call_ids":["senseval2.d002.s050.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reform%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028409999999999997,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.695989832980558}
{"item_id":"senseval2.d002.s050.t005","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval2.d002.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1487,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00028555,"input_uncached_usd":0.00007434999999999999,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9560566659783944}
{"item_id":"senseval2.d002.s050.t006","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval2.d002.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1486,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002599,"input_uncached_usd":0.00007429999999999999,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2186737079755403}
{"item_id":"senseval2.d002.s050.t007","gold_sense_keys":["parental%3:00:00::"],"candidates":[{"index":1,"sense_key":"parental%3:01:00::","synset_id":"parental.a.01"},{"index":2,"sense_key":"parental%3:00:00::","synset_id":"parental.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"parental%3:00:00::","call_ids":["senseval2.d002.s050.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"parental%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033279999999999996,"input_uncached_usd":0.0000192,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.2496333749732}
{"item_id":"senseval2.d002.s050.t008","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":3,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":4,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involvement%1:04:00::","call_ids":["senseval2.d002.s050.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028865,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.5885251659783535}
{"item_id":"senseval2.d002.s050.t009","gold_sense_keys":["bad%3:00:00::"],"candidates":[{"index":1,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":2,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":3,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":4,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":5,"sense_key":"bad%3:00:09::","synset_id":"regretful.a.01"},{"index":6,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":7,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":8,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":9,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":10,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":11,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":12,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":13,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":14,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bad%3:00:00::","call_ids":["senseval2.d002.s050.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046164999999999997,"input_uncached_usd":0.00004565,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.185941083997022}
{"item_id":"senseval2.d002.s050.t010","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"idea%1:09:03::","call_ids":["senseval2.d002.s050.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"idea%1:09:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00067415,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.919679292012006}
{"item_id":"senseval2.d002.s051.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s051.t000__v1__a1","senseval2.d002.s051.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005130499999999999,"input_uncached_usd":0.00003945,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.450437999970745}
{"item_id":"senseval2.d002.s051.t001","gold_sense_keys":["too%4:02:00::"],"candidates":[{"index":1,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"},{"index":2,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"too%4:02:00::","call_ids":["senseval2.d002.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"too%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040915,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.426352958951611}
{"item_id":"senseval2.d002.s051.t002","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"likely%3:00:04::","call_ids":["senseval2.d002.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"likely%3:00:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.000695,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.678657207987271}
{"item_id":"senseval2.d002.s051.t003","gold_sense_keys":["blame%2:31:00::","blame%2:32:00::"],"candidates":[{"index":1,"sense_key":"blame%2:32:00::","synset_id":"blame.v.01"},{"index":2,"sense_key":"blame%2:32:01::","synset_id":"blame.v.02"},{"index":3,"sense_key":"blame%2:31:00::","synset_id":"blame.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"blame%2:31:00::","call_ids":["senseval2.d002.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"blame%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020585,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.629475000023376}
{"item_id":"senseval2.d002.s051.t004","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005735,"input_uncached_usd":0.0000295,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.387612167047337}
{"item_id":"senseval2.d002.s051.t005","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s051.t005__v1__a1","senseval2.d002.s051.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046055,"input_uncached_usd":0.00003814999999999999,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.247753749019466}
{"item_id":"senseval2.d002.s051.t006","gold_sense_keys":["limitation%1:07:00::"],"candidates":[{"index":1,"sense_key":"limitation%1:09:00::","synset_id":"restriction.n.01"},{"index":2,"sense_key":"limitation%1:07:00::","synset_id":"limitation.n.02"},{"index":3,"sense_key":"limitation%1:23:00::","synset_id":"limit.n.06"},{"index":4,"sense_key":"limitation%1:28:00::","synset_id":"limitation.n.04"},{"index":5,"sense_key":"limitation%1:04:00::","synset_id":"limitation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"limitation%1:07:00::","call_ids":["senseval2.d002.s051.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"limitation%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026165,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.493806708022021}
{"item_id":"senseval2.d002.s051.t007","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s051.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018334999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.821003541990649}
{"item_id":"senseval2.d002.s052.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005122000000000001,"input_uncached_usd":0.0000194,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.773867457988672}
{"item_id":"senseval2.d002.s052.t001","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["senseval2.d002.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011069999999999998,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9886945839971304}
{"item_id":"senseval2.d002.s052.t002","gold_sense_keys":["child%1:18:01::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002611,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.73271862504771}
{"item_id":"senseval2.d002.s052.t003","gold_sense_keys":["education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005217,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.4967810829984955}
{"item_id":"senseval2.d002.s052.t004","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval2.d002.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047774999999999993,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.9553362089791335}
{"item_id":"senseval2.d002.s053.t000","gold_sense_keys":["see%2:31:03::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"see%2:31:03::","call_ids":["senseval2.d002.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"see%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1450,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00023249999999999999,"input_uncached_usd":0.0000725,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2269905420253053}
{"item_id":"senseval2.d002.s053.t001","gold_sense_keys":["kid%1:18:01::"],"candidates":[{"index":1,"sense_key":"kid%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"kid%1:27:00::","synset_id":"kid.n.02"},{"index":3,"sense_key":"kid%1:18:02::","synset_id":"kyd.n.01"},{"index":4,"sense_key":"kid%1:18:01::","synset_id":"child.n.02"},{"index":5,"sense_key":"kid%1:05:00::","synset_id":"kid.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"kid%1:18:00::","call_ids":["senseval2.d002.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"kid%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002874,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.513814042031299}
{"item_id":"senseval2.d002.s053.t002","gold_sense_keys":["play%2:41:02::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"play%2:41:00::","call_ids":["senseval2.d002.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"play%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1843,"cached_input_tokens":0,"output_tokens":2065,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00091815,"input_uncached_usd":0.00009214999999999999,"input_cached_usd":0.0,"output_usd":0.0008259999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.413562332978472}
{"item_id":"senseval2.d002.s053.t006","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"child%1:18:01::","call_ids":["senseval2.d002.s053.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"child%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002608,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.322043832973577}
{"item_id":"senseval2.d002.s053.t007","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval2.d002.s053.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002829,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.790515959030017}
{"item_id":"senseval2.d002.s053.t009","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval2.d002.s053.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032355000000000003,"input_uncached_usd":0.00003555,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.995038792025298}
{"item_id":"senseval2.d002.s053.t011","gold_sense_keys":["scrutinize%2:39:00::"],"candidates":[{"index":1,"sense_key":"scrutinize%2:39:00::","synset_id":"size_up.v.01"},{"index":2,"sense_key":"scrutinize%2:31:00::","synset_id":"audit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scrutinize%2:31:00::","call_ids":["senseval2.d002.s053.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scrutinize%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.00097395,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.740620166005101}
{"item_id":"senseval2.d002.s054.t000","gold_sense_keys":["parent%1:18:00::"],"candidates":[{"index":1,"sense_key":"parent%1:18:00::","synset_id":"parent.n.01"},{"index":2,"sense_key":"parent%1:03:00::","synset_id":"parent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parent%1:18:00::","call_ids":["senseval2.d002.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040925,"input_uncached_usd":0.00001885,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.255034749978222}
{"item_id":"senseval2.d002.s054.t002","gold_sense_keys":["option%1:09:00::"],"candidates":[{"index":1,"sense_key":"option%1:21:00::","synset_id":"option.n.01"},{"index":2,"sense_key":"option%1:09:00::","synset_id":"option.n.02"},{"index":3,"sense_key":"option%1:04:00::","synset_id":"choice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"option%1:09:00::","call_ids":["senseval2.d002.s054.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"option%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.000387,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.131798917020205}
{"item_id":"senseval2.d002.s054.t003","gold_sense_keys":["switch%2:30:02::"],"candidates":[{"index":1,"sense_key":"switch%2:30:03::","synset_id":"switch_over.v.01"},{"index":2,"sense_key":"switch%2:40:00::","synset_id":"trade.v.04"},{"index":3,"sense_key":"switch%2:30:02::","synset_id":"switch.v.03"},{"index":4,"sense_key":"switch%2:30:01::","synset_id":"switch.v.04"},{"index":5,"sense_key":"switch%2:35:00::","synset_id":"throw.v.06"},{"index":6,"sense_key":"switch%2:35:03::","synset_id":"switch.v.06"},{"index":7,"sense_key":"switch%2:30:00::","synset_id":"interchange.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"switch%2:30:02::","call_ids":["senseval2.d002.s054.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"switch%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003425,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.7167564170085825}
{"item_id":"senseval2.d002.s055.t000","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":2,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":3,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":4,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":5,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":6,"sense_key":"community%1:14:02::","synset_id":"community.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"community%1:14:04::","call_ids":["senseval2.d002.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"community%1:14:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036454999999999994,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.516287167032715}
{"item_id":"senseval2.d002.s055.t002","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013145,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2582572920364328}
{"item_id":"senseval2.d002.s055.t003","gold_sense_keys":["worse%3:00:00::"],"candidates":[{"index":1,"sense_key":"worse%3:00:00::","synset_id":"worse.a.01"},{"index":2,"sense_key":"worse%3:00:02::","synset_id":"worse.a.02"},{"index":3,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":4,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":5,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":6,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":7,"sense_key":"regretful%3:00:00::","synset_id":"regretful.a.01"},{"index":8,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":9,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":10,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":11,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":12,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":13,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":14,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":15,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":16,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worse%3:00:00::","call_ids":["senseval2.d002.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worse%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1022,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00015989999999999998,"input_uncached_usd":0.000051099999999999995,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0513059169752523}
{"item_id":"senseval2.d002.s055.t004","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"idea%1:09:03::","call_ids":["senseval2.d002.s055.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"idea%1:09:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016089999999999998,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.40869000001112}
{"item_id":"senseval2.d002.s056.t000","gold_sense_keys":["here%4:02:02::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"here%4:02:02::","call_ids":["senseval2.d002.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"here%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033595,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0305862500099465}
{"item_id":"senseval2.d002.s057.t000","gold_sense_keys":["locally%4:02:00::"],"candidates":[{"index":1,"sense_key":"locally%4:02:00::","synset_id":"locally.r.01"},{"index":2,"sense_key":"locally%4:02:01::","synset_id":"locally.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"locally%4:02:00::","call_ids":["senseval2.d002.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"locally%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040925,"input_uncached_usd":0.00001885,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.495393791003153}
{"item_id":"senseval2.d002.s057.t003","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["senseval2.d002.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048679999999999996,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.943695833033416}
{"item_id":"senseval2.d002.s057.t005","gold_sense_keys":["city%1:15:00::"],"candidates":[{"index":1,"sense_key":"city%1:15:00::","synset_id":"city.n.01"},{"index":2,"sense_key":"city%1:15:01::","synset_id":"city.n.02"},{"index":3,"sense_key":"city%1:14:00::","synset_id":"city.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"city%1:15:01::","call_ids":["senseval2.d002.s057.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"city%1:15:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004888,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.837870875024237}
{"item_id":"senseval2.d002.s057.t006","gold_sense_keys":["become%2:42:01::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval2.d002.s057.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004652999999999999,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.3846499170176685}
{"item_id":"senseval2.d002.s057.t007","gold_sense_keys":["ambitious%3:00:00::"],"candidates":[{"index":1,"sense_key":"ambitious%3:00:00::","synset_id":"ambitious.a.01"},{"index":2,"sense_key":"ambitious%5:00:00:difficult:00","synset_id":"ambitious.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ambitious%3:00:00::","call_ids":["senseval2.d002.s057.t007__v1__a1","senseval2.d002.s057.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ambitious%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":2080,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00087285,"input_uncached_usd":0.000040850000000000004,"input_cached_usd":0.0,"output_usd":0.000832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.290920416009612}
{"item_id":"senseval2.d002.s057.t008","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":2,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":3,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generally%4:02:02::","call_ids":["senseval2.d002.s057.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018214999999999998,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1342127499519847}
{"item_id":"senseval2.d002.s057.t009","gold_sense_keys":["corrupt%3:00:00::","corrupt%3:00:04::"],"candidates":[{"index":1,"sense_key":"corrupt%3:00:00::","synset_id":"corrupt.a.01"},{"index":2,"sense_key":"corrupt%3:00:04::","synset_id":"crooked.a.02"},{"index":3,"sense_key":"corrupt%5:00:00:imperfect:00","synset_id":"corrupt.s.01"},{"index":4,"sense_key":"corrupt%5:00:00:stale:00","synset_id":"corrupt.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corrupt%5:00:00:imperfect:00","call_ids":["senseval2.d002.s057.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corrupt%5:00:00:imperfect:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006959,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.599424667016137}
{"item_id":"senseval2.d002.s057.t012","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval2.d002.s057.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000207,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7119810829753987}
{"item_id":"senseval2.d002.s057.t013","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s057.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033365,"input_uncached_usd":0.00002005,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.28050895902561}
{"item_id":"senseval2.d002.s057.t015","gold_sense_keys":["politician%1:18:00::","politician%1:18:01::"],"candidates":[{"index":1,"sense_key":"politician%1:18:01::","synset_id":"politician.n.01"},{"index":2,"sense_key":"politician%1:18:00::","synset_id":"politician.n.02"},{"index":3,"sense_key":"politician%1:18:02::","synset_id":"politician.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"politician%1:18:00::","call_ids":["senseval2.d002.s057.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"politician%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043599999999999997,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.055772457970306}
{"item_id":"senseval2.d002.s058.t000","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":2,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":3,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":4,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":5,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":6,"sense_key":"process%1:03:00::","synset_id":"process.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"process%1:04:00::","call_ids":["senseval2.d002.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"process%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047299999999999995,"input_uncached_usd":0.000031,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.583885959000327}
{"item_id":"senseval2.d002.s058.t001","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval2.d002.s058.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042845000000000003,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.84000229195226}
{"item_id":"senseval2.d002.s058.t002","gold_sense_keys":["system%1:09:00::"],"candidates":[{"index":1,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":2,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":3,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":4,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":5,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":7,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":8,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":9,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"system%1:09:02::","call_ids":["senseval2.d002.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"system%1:09:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022414999999999997,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.499250499997288}
{"item_id":"senseval2.d002.s058.t003","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s058.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004215,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.231427374994382}
{"item_id":"senseval2.d002.s058.t004","gold_sense_keys":["governance%1:04:00::"],"candidates":[{"index":1,"sense_key":"governance%1:14:00::","synset_id":"administration.n.02"},{"index":2,"sense_key":"governance%1:04:00::","synset_id":"government.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"governance%1:14:00::","call_ids":["senseval2.d002.s058.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"governance%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020815,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2913696659961715}
{"item_id":"senseval2.d002.s058.t006","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s058.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036379999999999995,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.577083374955691}
{"item_id":"senseval2.d002.s058.t007","gold_sense_keys":["city%1:15:01::"],"candidates":[{"index":1,"sense_key":"city%1:15:00::","synset_id":"city.n.01"},{"index":2,"sense_key":"city%1:15:01::","synset_id":"city.n.02"},{"index":3,"sense_key":"city%1:14:00::","synset_id":"city.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"city%1:15:01::","call_ids":["senseval2.d002.s058.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"city%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004385,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.244558833015617}
{"item_id":"senseval2.d002.s058.t008","gold_sense_keys":["institute%2:36:01::"],"candidates":[{"index":1,"sense_key":"institute%2:36:01::","synset_id":"establish.v.02"},{"index":2,"sense_key":"institute%2:36:00::","synset_id":"institute.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"institute%2:36:01::","call_ids":["senseval2.d002.s058.t008__v1__a1","senseval2.d002.s058.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"institute%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":929,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041374999999999995,"input_uncached_usd":0.000042149999999999995,"input_cached_usd":0.0,"output_usd":0.0003716,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.114328540978022}
{"item_id":"senseval2.d002.s059.t000","gold_sense_keys":["most%3:00:02::"],"candidates":[{"index":1,"sense_key":"most%3:00:02::","synset_id":"most.a.01"},{"index":2,"sense_key":"most%3:00:01::","synset_id":"most.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%3:00:02::","call_ids":["senseval2.d002.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005397,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.555549833981786}
{"item_id":"senseval2.d002.s059.t002","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["senseval2.d002.s059.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005897500000000001,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.374651250021998}
{"item_id":"senseval2.d002.s059.t003","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039325,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.604036542004906}
{"item_id":"senseval2.d002.s059.t005","gold_sense_keys":["circumstance%1:26:01::","circumstance%1:26:02::"],"candidates":[{"index":1,"sense_key":"circumstance%1:26:01::","synset_id":"circumstance.n.01"},{"index":2,"sense_key":"circumstance%1:26:02::","synset_id":"context.n.02"},{"index":3,"sense_key":"circumstance%1:09:00::","synset_id":"circumstance.n.03"},{"index":4,"sense_key":"circumstance%1:11:00::","synset_id":"circumstance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"circumstance%1:26:02::","call_ids":["senseval2.d002.s059.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"circumstance%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010575,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.648775583016686}
{"item_id":"senseval2.d002.s059.t006","gold_sense_keys":["probably%4:02:00::"],"candidates":[{"index":1,"sense_key":"probably%4:02:00::","synset_id":"probably.r.01"},{"index":2,"sense_key":"probably%4:02:01::","synset_id":"credibly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"probably%4:02:00::","call_ids":["senseval2.d002.s059.t006__v1__a1","senseval2.d002.s059.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"probably%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":2208,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.00092705,"input_uncached_usd":0.000043849999999999995,"input_cached_usd":0.0,"output_usd":0.0008832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.2553239580011}
{"item_id":"senseval2.d002.s059.t007","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"make%2:36:08::","call_ids":["senseval2.d002.s059.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"make%2:36:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2260,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00047819999999999997,"input_uncached_usd":0.000113,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.290099082980305}
{"item_id":"senseval2.d002.s059.t008","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"thing%1:09:02::","call_ids":["senseval2.d002.s059.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"thing%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045099999999999996,"input_uncached_usd":0.000035,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.847911624994595}
{"item_id":"senseval2.d002.s059.t009","gold_sense_keys":["worse%3:00:00::"],"candidates":[{"index":1,"sense_key":"worse%3:00:00::","synset_id":"worse.a.01"},{"index":2,"sense_key":"worse%3:00:02::","synset_id":"worse.a.02"},{"index":3,"sense_key":"bad%3:00:00::","synset_id":"bad.a.01"},{"index":4,"sense_key":"bad%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":5,"sense_key":"bad%5:00:00:uncomfortable:00","synset_id":"bad.s.02"},{"index":6,"sense_key":"bad%5:00:00:stale:00","synset_id":"bad.s.03"},{"index":7,"sense_key":"regretful%3:00:00::","synset_id":"regretful.a.01"},{"index":8,"sense_key":"bad%5:00:00:invalid:00","synset_id":"bad.s.04"},{"index":9,"sense_key":"bad%5:00:00:inferior:02","synset_id":"bad.s.05"},{"index":10,"sense_key":"bad%5:00:00:nonstandard:03","synset_id":"bad.s.06"},{"index":11,"sense_key":"bad%5:00:02:unsound:00","synset_id":"bad.s.07"},{"index":12,"sense_key":"bad%5:00:00:unhealthy:00","synset_id":"bad.s.08"},{"index":13,"sense_key":"bad%5:00:00:harmful:00","synset_id":"bad.s.09"},{"index":14,"sense_key":"bad%5:00:00:evil:00","synset_id":"bad.s.10"},{"index":15,"sense_key":"bad%5:00:00:counterfeit:00","synset_id":"bad.s.11"},{"index":16,"sense_key":"bad%5:00:00:malfunctioning:00","synset_id":"bad.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worse%3:00:00::","call_ids":["senseval2.d002.s059.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worse%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036544999999999997,"input_uncached_usd":0.00005185,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.357619583024643}
{"item_id":"senseval2.d002.s059.t010","gold_sense_keys":["better%3:00:00::"],"candidates":[{"index":1,"sense_key":"better%3:00:00::","synset_id":"better.a.01"},{"index":2,"sense_key":"better%3:00:02::","synset_id":"better.a.02"},{"index":3,"sense_key":"better%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":4,"sense_key":"better%5:00:00:major:01","synset_id":"better.s.02"},{"index":5,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":8,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":9,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":10,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":11,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":12,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":13,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":14,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":15,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":16,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":17,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":18,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":19,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":20,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":21,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":22,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":23,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":24,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":25,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"},{"index":26,"sense_key":"well%3:00:01::","synset_id":"well.a.01"},{"index":27,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":28,"sense_key":"well%5:00:00:advisable:00","synset_id":"well.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"better%3:00:00::","call_ids":["senseval2.d002.s059.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"better%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1580,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005461999999999999,"input_uncached_usd":0.000079,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.644269999989774}
{"item_id":"senseval2.d002.s060.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reason%1:10:00::","call_ids":["senseval2.d002.s060.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00062525,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.768847792001907}
{"item_id":"senseval2.d002.s060.t001","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval2.d002.s060.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047279999999999995,"input_uncached_usd":0.0000312,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5595063749933615}
{"item_id":"senseval2.d002.s060.t002","gold_sense_keys":["education%1:04:00::","education%1:09:01::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s060.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031735,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.63970183301717}
{"item_id":"senseval2.d002.s060.t005","gold_sense_keys":["influence%1:19:00::"],"candidates":[{"index":1,"sense_key":"influence%1:07:00::","synset_id":"influence.n.01"},{"index":2,"sense_key":"influence%1:04:00::","synset_id":"influence.n.02"},{"index":3,"sense_key":"influence%1:09:00::","synset_id":"influence.n.03"},{"index":4,"sense_key":"influence%1:19:00::","synset_id":"influence.n.04"},{"index":5,"sense_key":"influence%1:18:00::","synset_id":"influence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"influence%1:19:00::","call_ids":["senseval2.d002.s060.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"influence%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015984999999999998,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7665941669838503}
{"item_id":"senseval2.d002.s060.t006","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"money%1:21:02::","call_ids":["senseval2.d002.s060.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"money%1:21:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002327,"input_uncached_usd":0.000021499999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7928385420236737}
{"item_id":"senseval2.d002.s061.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval2.d002.s061.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002576,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.17500608303817}
{"item_id":"senseval2.d002.s061.t001","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061555,"input_uncached_usd":0.00001995,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.099525749974418}
{"item_id":"senseval2.d002.s061.t003","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"tell%2:32:08::","call_ids":["senseval2.d002.s061.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"tell%2:32:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024465,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.263483875023667}
{"item_id":"senseval2.d002.s061.t005","gold_sense_keys":["even%4:02:01::"],"candidates":[{"index":1,"sense_key":"even%4:02:00::","synset_id":"even.r.01"},{"index":2,"sense_key":"even%4:02:03::","synset_id":"even.r.02"},{"index":3,"sense_key":"even%4:02:01::","synset_id":"even.r.03"},{"index":4,"sense_key":"even%4:02:04::","synset_id":"even.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"even%4:02:01::","call_ids":["senseval2.d002.s061.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"even%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021105,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2387366669718176}
{"item_id":"senseval2.d002.s061.t007","gold_sense_keys":["class%1:14:03::"],"candidates":[{"index":1,"sense_key":"class%1:14:00::","synset_id":"class.n.01"},{"index":2,"sense_key":"class%1:14:03::","synset_id":"class.n.02"},{"index":3,"sense_key":"class%1:14:01::","synset_id":"class.n.03"},{"index":4,"sense_key":"class%1:04:00::","synset_id":"course.n.01"},{"index":5,"sense_key":"class%1:14:05::","synset_id":"class.n.05"},{"index":6,"sense_key":"class%1:14:04::","synset_id":"class.n.06"},{"index":7,"sense_key":"class%1:14:02::","synset_id":"class.n.07"},{"index":8,"sense_key":"class%1:07:00::","synset_id":"class.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"class%1:14:03::","call_ids":["senseval2.d002.s061.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"class%1:14:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003217,"input_uncached_usd":0.0000337,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.171225292026065}
{"item_id":"senseval2.d002.s061.t008","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["senseval2.d002.s061.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021539999999999998,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.897265375009738}
{"item_id":"senseval2.d002.s061.t009","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s061.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015725,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3497234999667853}
{"item_id":"senseval2.d002.s061.t010","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"performance%1:04:03::","call_ids":["senseval2.d002.s061.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"performance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036849999999999996,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.655468166980427}
{"item_id":"senseval2.d002.s061.t011","gold_sense_keys":["pupil%1:18:00::","pupil%1:18:01::"],"candidates":[{"index":1,"sense_key":"pupil%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"pupil%1:08:00::","synset_id":"pupil.n.02"},{"index":3,"sense_key":"pupil%1:18:01::","synset_id":"schoolchild.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pupil%1:18:01::","call_ids":["senseval2.d002.s061.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pupil%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002845,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.139311000006273}
{"item_id":"senseval2.d002.s061.t013","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s061.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002833,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.505923832999542}
{"item_id":"senseval2.d002.s061.t014","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s061.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033905,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.154495541006327}
{"item_id":"senseval2.d002.s061.t015","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reason%1:10:00::","call_ids":["senseval2.d002.s061.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044679999999999996,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.394513249979354}
{"item_id":"senseval2.d002.s062.t000","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval2.d002.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003052,"input_uncached_usd":0.0000428,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.038595208025072}
{"item_id":"senseval2.d002.s062.t001","gold_sense_keys":["money%1:21:00::","money%1:21:02::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"money%1:21:02::","call_ids":["senseval2.d002.s062.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"money%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015849999999999998,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.758976834011264}
{"item_id":"senseval2.d002.s062.t002","gold_sense_keys":["flow%2:38:01::"],"candidates":[{"index":1,"sense_key":"flow%2:38:01::","synset_id":"flow.v.01"},{"index":2,"sense_key":"flow%2:38:00::","synset_id":"run.v.06"},{"index":3,"sense_key":"flow%2:38:02::","synset_id":"flow.v.03"},{"index":4,"sense_key":"flow%2:42:00::","synset_id":"flow.v.04"},{"index":5,"sense_key":"flow%2:35:04::","synset_id":"hang.v.05"},{"index":6,"sense_key":"flow%2:30:00::","synset_id":"flow.v.06"},{"index":7,"sense_key":"flow%2:29:00::","synset_id":"menstruate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flow%2:38:01::","call_ids":["senseval2.d002.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flow%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037089999999999996,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.154924791015219}
{"item_id":"senseval2.d002.s062.t007","gold_sense_keys":["pile%2:35:03::"],"candidates":[{"index":1,"sense_key":"pile%2:35:00::","synset_id":"stack.v.02"},{"index":2,"sense_key":"pile%2:38:00::","synset_id":"throng.v.01"},{"index":3,"sense_key":"pile%2:35:03::","synset_id":"pile.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pile%2:35:03::","call_ids":["senseval2.d002.s062.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pile%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023605,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.497737583995331}
{"item_id":"senseval2.d002.s062.t008","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d002.s062.t008__v1__a1","senseval2.d002.s062.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047205,"input_uncached_usd":0.00004965,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.052213708986528}
{"item_id":"senseval2.d002.s062.t009","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["senseval2.d002.s062.t009__v1__a1","senseval2.d002.s062.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047205,"input_uncached_usd":0.00004965,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.965986875002272}
{"item_id":"senseval2.d002.s062.t011","gold_sense_keys":["teacher%1:18:00::"],"candidates":[{"index":1,"sense_key":"teacher%1:18:00::","synset_id":"teacher.n.01"},{"index":2,"sense_key":"teacher%1:09:00::","synset_id":"teacher.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teacher%1:18:00::","call_ids":["senseval2.d002.s062.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teacher%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003096,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.714045458007604}
{"item_id":"senseval2.d002.s063.t000","gold_sense_keys":["mystery%1:09:00::"],"candidates":[{"index":1,"sense_key":"mystery%1:09:00::","synset_id":"mystery.n.01"},{"index":2,"sense_key":"mystery%1:10:00::","synset_id":"mystery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mystery%1:09:00::","call_ids":["senseval2.d002.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mystery%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033594999999999995,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.792372416995931}
{"item_id":"senseval2.d002.s063.t002","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fact%1:26:01::","call_ids":["senseval2.d002.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fact%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044205,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.45215308299521}
{"item_id":"senseval2.d002.s063.t003","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s063.t003__v1__a1","senseval2.d002.s063.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004914499999999999,"input_uncached_usd":0.00004345,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.155759292014409}
{"item_id":"senseval2.d002.s063.t004","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["senseval2.d002.s063.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028474999999999996,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.005160000000615}
{"item_id":"senseval2.d002.s063.t005","gold_sense_keys":["real%3:00:01::"],"candidates":[{"index":1,"sense_key":"real%3:00:00::","synset_id":"real.a.01"},{"index":2,"sense_key":"real%3:00:02::","synset_id":"real.a.02"},{"index":3,"sense_key":"real%5:00:00:serious:00","synset_id":"real.s.01"},{"index":4,"sense_key":"real%5:00:00:concrete:00","synset_id":"real.s.02"},{"index":5,"sense_key":"real%5:00:00:true:00","synset_id":"actual.s.02"},{"index":6,"sense_key":"real%3:00:01::","synset_id":"real.a.06"},{"index":7,"sense_key":"real%3:00:03::","synset_id":"substantial.a.03"},{"index":8,"sense_key":"real%5:00:00:tangible:02","synset_id":"real.s.04"},{"index":9,"sense_key":"real%5:00:02:realistic:00","synset_id":"veridical.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"real%3:00:01::","call_ids":["senseval2.d002.s063.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"real%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001738,"input_uncached_usd":0.000039399999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0872411660384387}
{"item_id":"senseval2.d002.s063.t006","gold_sense_keys":["increase%2:30:00::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%2:30:00::","call_ids":["senseval2.d002.s063.t006__v1__a1","senseval2.d002.s063.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007227500000000001,"input_uncached_usd":0.000044349999999999994,"input_cached_usd":0.0,"output_usd":0.0006784,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.170193458034191}
{"item_id":"senseval2.d002.s063.t007","gold_sense_keys":["sharply%4:02:03::"],"candidates":[{"index":1,"sense_key":"sharply%4:02:00::","synset_id":"aggressively.r.01"},{"index":2,"sense_key":"sharply%4:02:01::","synset_id":"sharply.r.02"},{"index":3,"sense_key":"sharply%4:02:02::","synset_id":"sharply.r.03"},{"index":4,"sense_key":"sharply%4:02:03::","synset_id":"precipitously.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sharply%4:02:03::","call_ids":["senseval2.d002.s063.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sharply%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002629,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.197687833046075}
{"item_id":"senseval2.d002.s063.t008","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval2.d002.s063.t008__v1__a1","senseval2.d002.s063.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00057035,"input_uncached_usd":0.000045549999999999996,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.979341416968964}
{"item_id":"senseval2.d002.s063.t009","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["senseval2.d002.s063.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002696,"input_uncached_usd":0.0000328,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.703995292016771}
{"item_id":"senseval2.d002.s063.t010","gold_sense_keys":["spend%2:40:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spend%2:40:00::","call_ids":["senseval2.d002.s063.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spend%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018255,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1085368749918416}
{"item_id":"senseval2.d002.s063.t011","gold_sense_keys":["more%4:02:01::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"more%4:02:01::","call_ids":["senseval2.d002.s063.t011__v1__a1","senseval2.d002.s063.t011__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"more%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006983499999999999,"input_uncached_usd":0.000045549999999999996,"input_cached_usd":0.0,"output_usd":0.0006528,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.529986999928951}
{"item_id":"senseval2.d002.s063.t012","gold_sense_keys":["pupil%1:18:00::","pupil%1:18:01::"],"candidates":[{"index":1,"sense_key":"pupil%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"pupil%1:08:00::","synset_id":"pupil.n.02"},{"index":3,"sense_key":"pupil%1:18:01::","synset_id":"schoolchild.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pupil%1:18:01::","call_ids":["senseval2.d002.s063.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pupil%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025945,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.277886208030395}
{"item_id":"senseval2.d002.s063.t013","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval2.d002.s063.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031405,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.841068250010721}
{"item_id":"senseval2.d002.s063.t014","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["senseval2.d002.s063.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008298,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.876383624970913}
{"item_id":"senseval2.d002.s063.t015","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s063.t015__v1__a1","senseval2.d002.s063.t015__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056825,"input_uncached_usd":0.00004345,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.495165707950946}
{"item_id":"senseval2.d002.s063.t016","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"performance%1:04:03::","call_ids":["senseval2.d002.s063.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"performance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002923,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.466446291015018}
{"item_id":"senseval2.d002.s063.t017","gold_sense_keys":["decline%2:30:01::"],"candidates":[{"index":1,"sense_key":"decline%2:30:01::","synset_id":"worsen.v.01"},{"index":2,"sense_key":"decline%2:40:00::","synset_id":"refuse.v.02"},{"index":3,"sense_key":"decline%2:32:00::","synset_id":"refuse.v.01"},{"index":4,"sense_key":"decline%2:30:00::","synset_id":"decline.v.04"},{"index":5,"sense_key":"decline%2:38:00::","synset_id":"decline.v.05"},{"index":6,"sense_key":"decline%2:38:03::","synset_id":"decline.v.06"},{"index":7,"sense_key":"decline%2:32:01::","synset_id":"decline.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decline%2:30:01::","call_ids":["senseval2.d002.s063.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decline%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024185,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4918549159774557}
{"item_id":"senseval2.d002.s064.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval2.d002.s064.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002769,"input_uncached_usd":0.000039699999999999996,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.550173082971014}
{"item_id":"senseval2.d002.s065.t000","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048764999999999995,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.591256582993083}
{"item_id":"senseval2.d002.s065.t001","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s065.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005380000000000001,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.244970667001326}
{"item_id":"senseval2.d002.s065.t002","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s065.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036209999999999997,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.390908125031274}
{"item_id":"senseval2.d002.s065.t003","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025695,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9413211250212044}
{"item_id":"senseval2.d002.s065.t004","gold_sense_keys":["allow%2:42:00::"],"candidates":[{"index":1,"sense_key":"allow%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"allow%2:32:00::","synset_id":"permit.v.01"},{"index":3,"sense_key":"allow%2:40:00::","synset_id":"allow.v.03"},{"index":4,"sense_key":"allow%2:31:01::","synset_id":"allow.v.04"},{"index":5,"sense_key":"allow%2:42:00::","synset_id":"leave.v.06"},{"index":6,"sense_key":"allow%2:31:00::","synset_id":"allow.v.06"},{"index":7,"sense_key":"allow%2:42:07::","synset_id":"admit.v.05"},{"index":8,"sense_key":"allow%2:41:01::","synset_id":"give_up.v.11"},{"index":9,"sense_key":"allow%2:40:01::","synset_id":"allow.v.09"},{"index":10,"sense_key":"allow%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"allow%2:42:00::","call_ids":["senseval2.d002.s065.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"allow%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":832,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030399999999999996,"input_uncached_usd":0.000041599999999999995,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.933928708021995}
{"item_id":"senseval2.d002.s066.t000","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s066.t000__v1__a1","senseval2.d002.s066.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00075005,"input_uncached_usd":0.000046049999999999994,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.526436457992531}
{"item_id":"senseval2.d002.s066.t001","gold_sense_keys":["permit%2:32:00::"],"candidates":[{"index":1,"sense_key":"permit%2:32:00::","synset_id":"permit.v.01"},{"index":2,"sense_key":"permit%2:41:00::","synset_id":"let.v.01"},{"index":3,"sense_key":"permit%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"permit%2:32:06::","call_ids":["senseval2.d002.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"permit%2:32:06::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033985,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.556510999973398}
{"item_id":"senseval2.d002.s066.t002","gold_sense_keys":["graduate%2:40:01::"],"candidates":[{"index":1,"sense_key":"graduate%2:40:00::","synset_id":"graduate.v.01"},{"index":2,"sense_key":"graduate%2:40:01::","synset_id":"graduate.v.02"},{"index":3,"sense_key":"graduate%2:30:00::","synset_id":"calibrate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"graduate%2:40:00::","call_ids":["senseval2.d002.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"graduate%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006446000000000001,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.294790582964197}
{"item_id":"senseval2.d002.s066.t004","gold_sense_keys":["master%2:31:01::"],"candidates":[{"index":1,"sense_key":"master%2:31:01::","synset_id":"master.v.01"},{"index":2,"sense_key":"master%2:33:00::","synset_id":"overcome.v.02"},{"index":3,"sense_key":"master%2:41:00::","synset_id":"dominate.v.03"},{"index":4,"sense_key":"master%2:31:00::","synset_id":"master.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"master%2:31:01::","call_ids":["senseval2.d002.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"master%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003149,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.719028958003037}
{"item_id":"senseval2.d002.s066.t005","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["senseval2.d002.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006567,"input_uncached_usd":0.0000359,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.316507790994365}
{"item_id":"senseval2.d002.s066.t006","gold_sense_keys":["prevail%2:42:01::"],"candidates":[{"index":1,"sense_key":"prevail%2:42:02::","synset_id":"predominate.v.01"},{"index":2,"sense_key":"prevail%2:42:01::","synset_id":"prevail.v.02"},{"index":3,"sense_key":"prevail%2:42:00::","synset_id":"prevail.v.03"},{"index":4,"sense_key":"prevail%2:33:00::","synset_id":"prevail.v.04"},{"index":5,"sense_key":"prevail%2:32:02::","synset_id":"prevail.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevail%2:42:02::","call_ids":["senseval2.d002.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevail%2:42:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007262000000000001,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.38116433401592}
{"item_id":"senseval2.d002.s066.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval2.d002.s066.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018759999999999998,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0921375830075704}
{"item_id":"senseval2.d002.s067.t000","gold_sense_keys":["mean%2:42:00::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mean%2:32:03::","call_ids":["senseval2.d002.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mean%2:32:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042275,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.505255833035335}
{"item_id":"senseval2.d002.s067.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["senseval2.d002.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015825,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.37509633298032}
{"item_id":"senseval2.d002.s067.t002","gold_sense_keys":["purpose%1:07:00::","purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026155,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.135828333965037}
{"item_id":"senseval2.d002.s067.t004","gold_sense_keys":["permit%2:41:00::"],"candidates":[{"index":1,"sense_key":"permit%2:32:00::","synset_id":"permit.v.01"},{"index":2,"sense_key":"permit%2:41:00::","synset_id":"let.v.01"},{"index":3,"sense_key":"permit%2:32:06::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"permit%2:32:06::","call_ids":["senseval2.d002.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"permit%2:32:06::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044114999999999996,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.4530487500014715}
{"item_id":"senseval2.d002.s067.t007","gold_sense_keys":["flourish%2:40:00::"],"candidates":[{"index":1,"sense_key":"flourish%2:30:01::","synset_id":"boom.v.05"},{"index":2,"sense_key":"flourish%2:40:00::","synset_id":"thrive.v.02"},{"index":3,"sense_key":"flourish%2:35:00::","synset_id":"brandish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flourish%2:40:00::","call_ids":["senseval2.d002.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flourish%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031195,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.711491333029699}
{"item_id":"senseval2.d002.s067.t008","gold_sense_keys":["clearly%4:02:00::"],"candidates":[{"index":1,"sense_key":"clearly%4:02:00::","synset_id":"clearly.r.01"},{"index":2,"sense_key":"clearly%4:02:01::","synset_id":"intelligibly.r.01"},{"index":3,"sense_key":"clearly%4:02:02::","synset_id":"distinctly.r.01"},{"index":4,"sense_key":"clearly%4:02:03::","synset_id":"clearly.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"clearly%4:02:00::","call_ids":["senseval2.d002.s067.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"clearly%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031335,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.090227167005651}
{"item_id":"senseval2.d002.s067.t009","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020644999999999998,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.75398766703438}
{"item_id":"senseval2.d002.s067.t010","gold_sense_keys":["less%4:02:00::"],"candidates":[{"index":1,"sense_key":"less%4:02:00::","synset_id":"less.r.01"},{"index":2,"sense_key":"less%4:02:01::","synset_id":"less.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%4:02:00::","call_ids":["senseval2.d002.s067.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":3474,"reasoning_output_tokens":3456},"cost":{"currency":"USD","total_usd":0.0014103,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.0013896,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.81645300000673}
{"item_id":"senseval2.d002.s067.t012","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval2.d002.s067.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1968,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006936,"input_uncached_usd":0.00009839999999999999,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.158600499969907}
{"item_id":"senseval2.d002.s067.t013","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["senseval2.d002.s067.t013__v1__a1","senseval2.d002.s067.t013__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006432499999999999,"input_uncached_usd":0.000041649999999999996,"input_cached_usd":0.0,"output_usd":0.0006015999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.603527750005014}
{"item_id":"senseval2.d002.s067.t014","gold_sense_keys":["study%1:09:03::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"study%1:09:03::","call_ids":["senseval2.d002.s067.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"study%1:09:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00060705,"input_uncached_usd":0.000037449999999999995,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.610816291999072}
{"item_id":"senseval2.d002.s067.t015","gold_sense_keys":["enter%2:33:00::","enter%2:38:00::","enter%2:42:00::"],"candidates":[{"index":1,"sense_key":"enter%2:38:00::","synset_id":"enter.v.01"},{"index":2,"sense_key":"enter%2:33:00::","synset_id":"enter.v.02"},{"index":3,"sense_key":"enter%2:41:06::","synset_id":"enroll.v.01"},{"index":4,"sense_key":"enter%2:42:00::","synset_id":"figure.v.02"},{"index":5,"sense_key":"enter%2:32:04::","synset_id":"record.v.01"},{"index":6,"sense_key":"enter%2:36:00::","synset_id":"enter.v.06"},{"index":7,"sense_key":"enter%2:41:00::","synset_id":"accede.v.02"},{"index":8,"sense_key":"enter%2:35:03::","synset_id":"insert.v.01"},{"index":9,"sense_key":"enter%2:30:10::","synset_id":"embark.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"enter%2:30:10::","call_ids":["senseval2.d002.s067.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"enter%2:30:10::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039965,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.243592292012181}
{"item_id":"senseval2.d002.s067.t017","gold_sense_keys":["world%1:14:01::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"world%1:17:02::","call_ids":["senseval2.d002.s067.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"world%1:17:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029895,"input_uncached_usd":0.00003655,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.442483208025806}
{"item_id":"senseval2.d002.s067.t018","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%1:04:01::","call_ids":["senseval2.d002.s067.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011795,"input_uncached_usd":0.00003475,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5096540829981677}
{"item_id":"senseval2.d002.s068.t000","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notion%1:09:02::","call_ids":["senseval2.d002.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049285,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.666708540986292}
{"item_id":"senseval2.d002.s069.t000","gold_sense_keys":["purpose%1:07:00::","purpose%1:09:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"purpose%1:09:00::","call_ids":["senseval2.d002.s069.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"purpose%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001828,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.483460334013216}
{"item_id":"senseval2.d002.s069.t001","gold_sense_keys":["education%1:04:00::"],"candidates":[{"index":1,"sense_key":"education%1:04:00::","synset_id":"education.n.01"},{"index":2,"sense_key":"education%1:09:00::","synset_id":"education.n.02"},{"index":3,"sense_key":"education%1:09:01::","synset_id":"education.n.03"},{"index":4,"sense_key":"education%1:04:01::","synset_id":"education.n.04"},{"index":5,"sense_key":"education%1:07:01::","synset_id":"education.n.05"},{"index":6,"sense_key":"education%1:14:00::","synset_id":"department_of_education.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"education%1:09:01::","call_ids":["senseval2.d002.s069.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"education%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039295,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.121668791980483}
{"item_id":"senseval2.d002.s069.t002","gold_sense_keys":["encourage%2:32:00::","encourage%2:37:00::"],"candidates":[{"index":1,"sense_key":"encourage%2:41:00::","synset_id":"promote.v.01"},{"index":2,"sense_key":"encourage%2:37:00::","synset_id":"encourage.v.02"},{"index":3,"sense_key":"encourage%2:32:00::","synset_id":"encourage.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"encourage%2:37:00::","call_ids":["senseval2.d002.s069.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"encourage%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018035,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.336495792027563}
{"item_id":"senseval2.d002.s069.t003","gold_sense_keys":["young%3:00:00::"],"candidates":[{"index":1,"sense_key":"young%3:00:00::","synset_id":"young.a.01"},{"index":2,"sense_key":"young%5:00:00:early:02","synset_id":"new.s.08"},{"index":3,"sense_key":"young%5:00:01:young:00","synset_id":"youthful.s.01"},{"index":4,"sense_key":"young%5:00:00:new:00","synset_id":"young.s.03"},{"index":5,"sense_key":"young%5:00:00:inexperienced:00","synset_id":"unseasoned.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"young%3:00:00::","call_ids":["senseval2.d002.s069.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"young%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044189999999999995,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.820201791997533}
{"item_id":"senseval2.d002.s069.t004","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"man%1:18:04::","call_ids":["senseval2.d002.s069.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"man%1:18:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00012484999999999997,"input_uncached_usd":0.000041649999999999996,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.161070749978535}
{"item_id":"senseval2.d002.s069.t005","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"woman%1:14:00::","call_ids":["senseval2.d002.s069.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"woman%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005431,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.899056541034952}
{"item_id":"senseval2.d002.s069.t006","gold_sense_keys":["realize%2:36:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"realize%2:36:00::","call_ids":["senseval2.d002.s069.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"realize%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024079999999999997,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.74313020799309}
{"item_id":"senseval2.d002.s069.t007","gold_sense_keys":["full%5:00:00:complete:00","full%5:00:01:whole:00"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"full%5:00:00:complete:00","call_ids":["senseval2.d002.s069.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"full%5:00:00:complete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042144999999999997,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.960414458008017}
{"item_id":"senseval2.d002.s069.t008","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s069.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015529999999999998,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7992337089963257}
{"item_id":"senseval2.d002.s069.t009","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s069.t009__v1__a1","senseval2.d002.s069.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058975,"input_uncached_usd":0.000039349999999999994,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.216684292070568}
{"item_id":"senseval2.d002.s070.t000","gold_sense_keys":["actually%4:02:01::"],"candidates":[{"index":1,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"},{"index":2,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":3,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":4,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"actually%4:02:03::","call_ids":["senseval2.d002.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"actually%4:02:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006967,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.78697362501407}
{"item_id":"senseval2.d002.s070.t001","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval2.d002.s070.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":1937,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00080145,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.0007748,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.970239833986852}
{"item_id":"senseval2.d002.s070.t002","gold_sense_keys":["equal%3:00:00::"],"candidates":[{"index":1,"sense_key":"equal%3:00:00::","synset_id":"equal.a.01"},{"index":2,"sense_key":"equal%3:00:03::","synset_id":"adequate.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equal%3:00:00::","call_ids":["senseval2.d002.s070.t002__v1__a1","senseval2.d002.s070.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046465,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.994238123937976}
{"item_id":"senseval2.d002.s070.t003","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s070.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015614999999999997,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.534578832972329}
{"item_id":"senseval2.d002.s070.t004","gold_sense_keys":["potential%1:26:00::"],"candidates":[{"index":1,"sense_key":"potential%1:26:00::","synset_id":"potential.n.01"},{"index":2,"sense_key":"potential%1:19:00::","synset_id":"electric_potential.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"potential%1:26:00::","call_ids":["senseval2.d002.s070.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"potential%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017935,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.215593541972339}
{"item_id":"senseval2.d002.s071.t000","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":2,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":3,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generally%4:02:02::","call_ids":["senseval2.d002.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generally%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.000183,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6781467079999857}
{"item_id":"senseval2.d002.s071.t001","gold_sense_keys":["desirable%3:00:00::"],"candidates":[{"index":1,"sense_key":"desirable%3:00:00::","synset_id":"desirable.a.01"},{"index":2,"sense_key":"desirable%5:00:00:eligible:00","synset_id":"desirable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"desirable%3:00:00::","call_ids":["senseval2.d002.s071.t001__v1__a1","senseval2.d002.s071.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"desirable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":849,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00051605,"input_uncached_usd":0.00004245,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.066599957935978}
{"item_id":"senseval2.d002.s071.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval2.d002.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016439999999999998,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.833096624992322}
{"item_id":"senseval2.d002.s071.t005","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"},{"index":2,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":3,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":4,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s071.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010849999999999998,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.034672917041462}
{"item_id":"senseval2.d002.s071.t006","gold_sense_keys":["new%3:00:00::"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"new%5:00:02:original:00","call_ids":["senseval2.d002.s071.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"new%5:00:02:original:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":1681,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007125499999999998,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.0006724,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.831199375039432}
{"item_id":"senseval2.d002.s071.t007","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s071.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004368,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.566945041995496}
{"item_id":"senseval2.d002.s072.t003","gold_sense_keys":["often%4:02:02::"],"candidates":[{"index":1,"sense_key":"often%4:02:00::","synset_id":"frequently.r.01"},{"index":2,"sense_key":"often%4:02:01::","synset_id":"much.r.05"},{"index":3,"sense_key":"often%4:02:02::","synset_id":"often.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"often%4:02:00::","call_ids":["senseval2.d002.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"often%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030920000000000003,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.094151541998144}
{"item_id":"senseval2.d002.s073.t000","gold_sense_keys":["dubious%5:00:00:questionable:00"],"candidates":[{"index":1,"sense_key":"dubious%5:00:00:uncertain:02","synset_id":"doubtful.s.02"},{"index":2,"sense_key":"dubious%5:00:00:questionable:00","synset_id":"doubtful.s.01"},{"index":3,"sense_key":"dubious%5:00:00:unconvinced:00","synset_id":"dubious.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dubious%5:00:00:questionable:00","call_ids":["senseval2.d002.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dubious%5:00:00:questionable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018255,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.901489291980397}
{"item_id":"senseval2.d002.s073.t001","gold_sense_keys":["psychological%3:01:00::"],"candidates":[{"index":1,"sense_key":"psychological%5:00:00:mental:00","synset_id":"psychological.s.01"},{"index":2,"sense_key":"psychological%3:01:00::","synset_id":"psychological.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"psychological%3:01:00::","call_ids":["senseval2.d002.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"psychological%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017849999999999997,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1141953330370598}
{"item_id":"senseval2.d002.s073.t004","gold_sense_keys":["rather%4:02:02::"],"candidates":[{"index":1,"sense_key":"rather%4:02:02::","synset_id":"rather.r.01"},{"index":2,"sense_key":"rather%4:02:04::","synset_id":"rather.r.02"},{"index":3,"sense_key":"rather%4:02:01::","synset_id":"preferably.r.01"},{"index":4,"sense_key":"rather%4:02:00::","synset_id":"quite.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rather%4:02:02::","call_ids":["senseval2.d002.s073.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rather%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023549999999999998,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.570752000028733}
{"item_id":"senseval2.d002.s073.t005","gold_sense_keys":["educational%3:01:00::"],"candidates":[{"index":1,"sense_key":"educational%3:01:00::","synset_id":"educational.a.01"},{"index":2,"sense_key":"educational%5:00:00:instructive:00","synset_id":"educational.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"educational%3:01:00::","call_ids":["senseval2.d002.s073.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"educational%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002293,"input_uncached_usd":0.0000181,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8604102919925936}
{"item_id":"senseval2.d002.s073.t006","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["senseval2.d002.s073.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028325,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.830825958983041}
{"item_id":"senseval2.d002.s074.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval2.d002.s074.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007286,"input_uncached_usd":0.000031,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.326829832978547}
{"item_id":"senseval2.d002.s074.t001","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041145,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.436144332983531}
{"item_id":"senseval2.d002.s074.t002","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s074.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028335,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.544529875041917}
{"item_id":"senseval2.d002.s074.t003","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"so%4:02:02::","call_ids":["senseval2.d002.s074.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"so%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038024999999999995,"input_uncached_usd":0.000041049999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.24067783303326}
{"item_id":"senseval2.d002.s074.t006","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["senseval2.d002.s074.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003902,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.175970042007975}
{"item_id":"senseval2.d002.s074.t011","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s074.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046249999999999997,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.892762542003766}
{"item_id":"senseval2.d002.s074.t012","gold_sense_keys":["influence%1:19:00::"],"candidates":[{"index":1,"sense_key":"influence%1:07:00::","synset_id":"influence.n.01"},{"index":2,"sense_key":"influence%1:04:00::","synset_id":"influence.n.02"},{"index":3,"sense_key":"influence%1:09:00::","synset_id":"influence.n.03"},{"index":4,"sense_key":"influence%1:19:00::","synset_id":"influence.n.04"},{"index":5,"sense_key":"influence%1:18:00::","synset_id":"influence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"influence%1:19:00::","call_ids":["senseval2.d002.s074.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"influence%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039115,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.673451040987857}
{"item_id":"senseval2.d002.s074.t013","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["senseval2.d002.s074.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028345,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.578795416979119}
{"item_id":"senseval2.d002.s075.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval2.d002.s075.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018734999999999997,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0396219580434263}
{"item_id":"senseval2.d002.s075.t001","gold_sense_keys":["appreciate%2:31:00::"],"candidates":[{"index":1,"sense_key":"appreciate%2:37:00::","synset_id":"appreciate.v.01"},{"index":2,"sense_key":"appreciate%2:31:00::","synset_id":"appreciate.v.02"},{"index":3,"sense_key":"appreciate%2:40:00::","synset_id":"prize.v.01"},{"index":4,"sense_key":"appreciate%2:30:00::","synset_id":"appreciate.v.04"},{"index":5,"sense_key":"appreciate%2:30:01::","synset_id":"appreciate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"appreciate%2:31:00::","call_ids":["senseval2.d002.s075.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"appreciate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004944,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.62488741602283}
{"item_id":"senseval2.d002.s075.t002","gold_sense_keys":["just%4:02:05::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"just%4:02:03::","call_ids":["senseval2.d002.s075.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"just%4:02:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037114999999999994,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.833213958016131}
{"item_id":"senseval2.d002.s075.t004","gold_sense_keys":["situation%1:26:00::"],"candidates":[{"index":1,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"},{"index":2,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":3,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":4,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":5,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"situation%1:26:00::","call_ids":["senseval2.d002.s075.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"situation%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029214999999999997,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.980387166026048}
{"item_id":"senseval2.d002.s075.t005","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064235,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.073223750048783}
{"item_id":"senseval2.d002.s075.t006","gold_sense_keys":["add%2:31:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"add%2:40:00::","call_ids":["senseval2.d002.s075.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"add%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004730499999999999,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1467585830250755}
{"item_id":"senseval2.d002.s075.t007","gold_sense_keys":["subtract%2:31:00::"],"candidates":[{"index":1,"sense_key":"subtract%2:31:00::","synset_id":"subtract.v.01"},{"index":2,"sense_key":"subtract%2:30:00::","synset_id":"subtract.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subtract%2:31:00::","call_ids":["senseval2.d002.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subtract%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003091,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.798912124999333}
{"item_id":"senseval2.d002.s075.t008","gold_sense_keys":["learn%2:31:00::"],"candidates":[{"index":1,"sense_key":"learn%2:31:00::","synset_id":"learn.v.01"},{"index":2,"sense_key":"learn%2:31:01::","synset_id":"learn.v.02"},{"index":3,"sense_key":"learn%2:31:03::","synset_id":"memorize.v.01"},{"index":4,"sense_key":"learn%2:31:02::","synset_id":"learn.v.04"},{"index":5,"sense_key":"learn%2:32:00::","synset_id":"teach.v.01"},{"index":6,"sense_key":"learn%2:32:02::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"learn%2:31:00::","call_ids":["senseval2.d002.s075.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"learn%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005511,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.150346792011987}
{"item_id":"senseval2.d002.s075.t010","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["senseval2.d002.s075.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018465,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.230455750017427}
{"item_id":"senseval2.d002.s075.t011","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["senseval2.d002.s075.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.623190125043038}
{"item_id":"senseval2.d002.s075.t013","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["senseval2.d002.s075.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004319,"input_uncached_usd":0.0000415,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.496527250041254}
{"item_id":"senseval2.d002.s075.t014","gold_sense_keys":["article%1:10:00::"],"candidates":[{"index":1,"sense_key":"article%1:10:00::","synset_id":"article.n.01"},{"index":2,"sense_key":"article%1:03:00::","synset_id":"article.n.02"},{"index":3,"sense_key":"article%1:10:02::","synset_id":"article.n.03"},{"index":4,"sense_key":"article%1:10:01::","synset_id":"article.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"article%1:10:00::","call_ids":["senseval2.d002.s075.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"article%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001583,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8239055420272052}
{"item_id":"senseval2.d002.s075.t015","gold_sense_keys":["recent%5:00:00:new:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recent%5:00:00:new:00","call_ids":["senseval2.d002.s075.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recent%5:00:00:new:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008213000000000001,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.732374167011585}
{"item_id":"senseval2.d002.s075.t017","gold_sense_keys":["major%1:18:01::"],"candidates":[{"index":1,"sense_key":"major%1:18:00::","synset_id":"major.n.01"},{"index":2,"sense_key":"major%1:18:02::","synset_id":"major.n.02"},{"index":3,"sense_key":"major%1:18:01::","synset_id":"major.n.03"},{"index":4,"sense_key":"major%1:09:00::","synset_id":"major.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"major%1:18:01::","call_ids":["senseval2.d002.s075.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"major%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023694999999999999,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.309718666016124}
{"item_id":"senseval2.d002.s076.t000","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d002.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046669999999999996,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.906002000032458}
{"item_id":"senseval2.d002.s076.t001","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"important%3:00:04::","call_ids":["senseval2.d002.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"important%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00065045,"input_uncached_usd":0.00002965,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.736817250028253}
{"item_id":"senseval2.d002.s076.t002","gold_sense_keys":["school%1:14:00::","school%1:14:03::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s076.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037239999999999994,"input_uncached_usd":0.0000332,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0741183329955675}
{"item_id":"senseval2.d002.s076.t003","gold_sense_keys":["principal%1:18:00::"],"candidates":[{"index":1,"sense_key":"principal%1:21:01::","synset_id":"principal.n.01"},{"index":2,"sense_key":"principal%1:18:00::","synset_id":"principal.n.02"},{"index":3,"sense_key":"principal%1:18:01::","synset_id":"star.n.04"},{"index":4,"sense_key":"principal%1:21:02::","synset_id":"principal.n.04"},{"index":5,"sense_key":"principal%1:18:03::","synset_id":"principal.n.05"},{"index":6,"sense_key":"principal%1:18:02::","synset_id":"principal.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"principal%1:18:00::","call_ids":["senseval2.d002.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"principal%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047089999999999995,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.832985417044256}
{"item_id":"senseval2.d002.s076.t005","gold_sense_keys":["measure%1:03:00::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"measure%1:03:00::","call_ids":["senseval2.d002.s076.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"measure%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006079500000000001,"input_uncached_usd":0.00003835,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.639391791948583}
{"item_id":"senseval2.d002.s076.t006","gold_sense_keys":["authority%1:07:00::"],"candidates":[{"index":1,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":2,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"},{"index":3,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":4,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":5,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":6,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":7,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"authority%1:07:00::","call_ids":["senseval2.d002.s076.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"authority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003225,"input_uncached_usd":0.0000345,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.660719666979276}
{"item_id":"senseval2.d002.s076.t007","gold_sense_keys":["faculty%1:14:00::"],"candidates":[{"index":1,"sense_key":"faculty%1:09:00::","synset_id":"faculty.n.01"},{"index":2,"sense_key":"faculty%1:14:00::","synset_id":"staff.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"faculty%1:14:00::","call_ids":["senseval2.d002.s076.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"faculty%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046399999999999995,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.215459041995928}
{"item_id":"senseval2.d002.s076.t009","gold_sense_keys":["matter%1:09:01::"],"candidates":[{"index":1,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":2,"sense_key":"matter%1:09:00::","synset_id":"topic.n.02"},{"index":3,"sense_key":"matter%1:03:00::","synset_id":"matter.n.03"},{"index":4,"sense_key":"matter%1:09:02::","synset_id":"matter.n.04"},{"index":5,"sense_key":"matter%1:07:00::","synset_id":"matter.n.05"},{"index":6,"sense_key":"matter%1:10:00::","synset_id":"matter.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"matter%1:09:00::","call_ids":["senseval2.d002.s076.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"matter%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047209999999999993,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.192468458029907}
{"item_id":"senseval2.d002.s076.t010","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s076.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023365,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.10734708304517}
{"item_id":"senseval2.d002.s077.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["senseval2.d002.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032655,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.477212833997328}
{"item_id":"senseval2.d002.s077.t001","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["senseval2.d002.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017294999999999998,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1837635000119917}
{"item_id":"senseval2.d002.s077.t002","gold_sense_keys":["most%4:02:00::"],"candidates":[{"index":1,"sense_key":"most%4:02:00::","synset_id":"most.r.01"},{"index":2,"sense_key":"most%4:02:01::","synset_id":"most.r.02"},{"index":3,"sense_key":"most%4:02:02::","synset_id":"about.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"most%4:02:00::","call_ids":["senseval2.d002.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"most%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044054999999999995,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.402593083970714}
{"item_id":"senseval2.d002.s077.t004","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval2.d002.s077.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007322,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.485753083019517}
{"item_id":"senseval2.d002.s077.t005","gold_sense_keys":["best%3:00:00::"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"good%3:00:01::","call_ids":["senseval2.d002.s077.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"good%3:00:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1396,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003834,"input_uncached_usd":0.0000698,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.171755917021073}
{"item_id":"senseval2.d002.s077.t006","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"school%1:14:00::","call_ids":["senseval2.d002.s077.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"school%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029505,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.54391258303076}
{"item_id":"senseval2.d002.s077.t007","gold_sense_keys":["outside%5:00:00:external:00","outside%5:00:00:extrinsic:00"],"candidates":[{"index":1,"sense_key":"outside%3:00:00::","synset_id":"outside.a.01"},{"index":2,"sense_key":"outside%5:00:00:extrinsic:00","synset_id":"external.s.01"},{"index":3,"sense_key":"outside%5:00:00:external:00","synset_id":"outside.s.02"},{"index":4,"sense_key":"outside%3:00:04::","synset_id":"outdoor.a.01"},{"index":5,"sense_key":"outside%5:00:00:extramural:00","synset_id":"outside.s.03"},{"index":6,"sense_key":"outside%5:00:00:exterior:00","synset_id":"outside.s.04"},{"index":7,"sense_key":"outside%5:00:00:foreign:02","synset_id":"external.s.02"},{"index":8,"sense_key":"outside%5:00:00:unlikely:00","synset_id":"outside.s.06"},{"index":9,"sense_key":"outside%5:00:00:outer:00","synset_id":"outside.s.07"},{"index":10,"sense_key":"outside%5:00:00:inaccurate:00","synset_id":"away.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"outside%5:00:00:external:00","call_ids":["senseval2.d002.s077.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"outside%5:00:00:external:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045489999999999994,"input_uncached_usd":0.0000389,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.134971374995075}
{"item_id":"senseval2.d002.s077.t008","gold_sense_keys":["interference%1:04:00::"],"candidates":[{"index":1,"sense_key":"interference%1:10:00::","synset_id":"intervention.n.02"},{"index":2,"sense_key":"interference%1:04:00::","synset_id":"hindrance.n.03"},{"index":3,"sense_key":"interference%1:11:00::","synset_id":"noise.n.03"},{"index":4,"sense_key":"interference%1:04:01::","synset_id":"interference.n.04"},{"index":5,"sense_key":"interference%1:06:00::","synset_id":"hindrance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"interference%1:04:00::","call_ids":["senseval2.d002.s077.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"interference%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049435,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.969748208008241}
{"item_id":"senseval2.d002.s077.t009","gold_sense_keys":["govern%2:41:00::"],"candidates":[{"index":1,"sense_key":"govern%2:41:02::","synset_id":"regulate.v.02"},{"index":2,"sense_key":"govern%2:41:01::","synset_id":"govern.v.02"},{"index":3,"sense_key":"govern%2:41:00::","synset_id":"govern.v.03"},{"index":4,"sense_key":"govern%2:42:00::","synset_id":"govern.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"govern%2:41:00::","call_ids":["senseval2.d002.s077.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"govern%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023794999999999998,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6863412500242703}
{"item_id":"senseval2.d002.s077.t010","gold_sense_keys":["powerful%3:00:00::","powerful%5:00:00:influential:00"],"candidates":[{"index":1,"sense_key":"powerful%3:00:00::","synset_id":"powerful.a.01"},{"index":2,"sense_key":"powerful%5:00:02:strong:00","synset_id":"knock-down.s.01"},{"index":3,"sense_key":"powerful%5:00:00:influential:00","synset_id":"potent.s.01"},{"index":4,"sense_key":"powerful%5:00:01:strong:00","synset_id":"brawny.s.01"},{"index":5,"sense_key":"powerful%5:00:00:superhuman:00","synset_id":"herculean.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"powerful%5:00:00:influential:00","call_ids":["senseval2.d002.s077.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"powerful%5:00:00:influential:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013719999999999997,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7830918330000713}
{"item_id":"senseval2.d002.s077.t011","gold_sense_keys":["head%1:18:01::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"head%1:18:01::","call_ids":["senseval2.d002.s077.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"head%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1640,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0005235999999999999,"input_uncached_usd":0.000082,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.6559473749948665}
{"item_id":"senseval2.d002.s078.t000","gold_sense_keys":["authority%1:07:00::"],"candidates":[{"index":1,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":2,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"},{"index":3,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":4,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":5,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":6,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":7,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"authority%1:07:00::","call_ids":["senseval2.d002.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"authority%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027059999999999996,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7969690829631872}
{"item_id":"senseval2.d002.s078.t002","gold_sense_keys":["go_with%2:42:01::"],"candidates":[{"index":1,"sense_key":"go_with%2:42:01::","synset_id":"attach_to.v.01"},{"index":2,"sense_key":"go_with%2:42:00::","synset_id":"collocate_with.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go_with%2:42:01::","call_ids":["senseval2.d002.s078.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go_with%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00077205,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.838618125009816}
{"item_id":"senseval2.d002.s078.t003","gold_sense_keys":["unambiguous%3:00:04::"],"candidates":[{"index":1,"sense_key":"unambiguous%3:00:00::","synset_id":"unambiguous.a.01"},{"index":2,"sense_key":"unambiguous%3:00:04::","synset_id":"unequivocal.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"unambiguous%3:00:04::","call_ids":["senseval2.d002.s078.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"unambiguous%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003625999999999999,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.778019374993164}
{"item_id":"senseval2.d002.s079.t000","gold_sense_keys":["school%1:14:00::"],"candidates":[{"index":1,"sense_key":"school%1:14:00::","synset_id":"school.n.01"},{"index":2,"sense_key":"school%1:06:00::","synset_id":"school.n.02"},{"index":3,"sense_key":"school%1:09:00::","synset_id":"school.n.03"},{"index":4,"sense_key":"school%1:14:02::","synset_id":"school.n.04"},{"index":5,"sense_key":"school%1:28:00::","synset_id":"school.n.05"},{"index":6,"sense_key":"school%1:14:03::","synset_id":"school.n.06"},{"index":7,"sense_key":"school%1:14:01::","synset_id":"school.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"school%1:14:03::","call_ids":["senseval2.d002.s079.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"school%1:14:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.000347,"input_uncached_usd":0.0000334,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.306701374996919}
{"item_id":"senseval2.d002.s079.t002","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval2.d002.s079.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":843,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017654999999999998,"input_uncached_usd":0.00004215,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.750212958024349}
{"item_id":"senseval2.d002.s079.t003","gold_sense_keys":["produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"produce%2:36:03::","call_ids":["senseval2.d002.s079.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"produce%2:36:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047719999999999994,"input_uncached_usd":0.0000356,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.17021129198838}
{"item_id":"senseval2.d002.s079.t004","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["senseval2.d002.s079.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018305,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2576995409908704}
{"item_id":"senseval2.d002.s079.t005","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval2.d002.s079.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001673,"input_uncached_usd":0.0000325,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.635982125008013}
{"item_id":"senseval2.d002.s079.t006","gold_sense_keys":["morale%1:26:00::"],"candidates":[{"index":1,"sense_key":"morale%1:26:00::","synset_id":"morale.n.01"},{"index":2,"sense_key":"morale%1:07:00::","synset_id":"esprit_de_corps.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morale%1:26:00::","call_ids":["senseval2.d002.s079.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morale%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005668499999999999,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.197313999990001}
{"item_id":"senseval2.d002.s079.t007","gold_sense_keys":["superior%3:00:02::"],"candidates":[{"index":1,"sense_key":"superior%3:00:02::","synset_id":"superior.a.01"},{"index":2,"sense_key":"superior%3:00:01::","synset_id":"superior.a.02"},{"index":3,"sense_key":"superior%5:00:00:dominant:01","synset_id":"superior.s.01"},{"index":4,"sense_key":"superior%3:00:00::","synset_id":"superscript.a.01"},{"index":5,"sense_key":"superior%3:00:03::","synset_id":"superior.a.05"},{"index":6,"sense_key":"superior%5:00:00:senior:00","synset_id":"ranking.s.01"},{"index":7,"sense_key":"superior%5:00:00:unaffected:00","synset_id":"superior.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"superior%3:00:02::","call_ids":["senseval2.d002.s079.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"superior%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039824999999999995,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.010285124997608}
{"item_id":"senseval2.d002.s079.t008","gold_sense_keys":["academic%3:01:00::"],"candidates":[{"index":1,"sense_key":"academic%3:01:00::","synset_id":"academic.a.01"},{"index":2,"sense_key":"academic%5:00:00:theoretical:01","synset_id":"academic.s.01"},{"index":3,"sense_key":"academic%5:00:00:scholarly:00","synset_id":"academic.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"academic%3:01:00::","call_ids":["senseval2.d002.s079.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"academic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002103,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.969831415975932}
{"item_id":"senseval2.d002.s079.t009","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"performance%1:04:00::","call_ids":["senseval2.d002.s079.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"performance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044709999999999997,"input_uncached_usd":0.0000311,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.773716583033092}
{"item_id":"senseval2.d002.s080.t000","gold_sense_keys":["fact%1:10:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fact%1:10:01::","call_ids":["senseval2.d002.s080.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fact%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005192,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.557530582998879}
{"item_id":"senseval2.d002.s080.t001","gold_sense_keys":["ruffle%2:37:01::"],"candidates":[{"index":1,"sense_key":"ruffle%2:38:00::","synset_id":"ripple.v.01"},{"index":2,"sense_key":"ruffle%2:37:00::","synset_id":"ruffle.v.02"},{"index":3,"sense_key":"ruffle%2:38:01::","synset_id":"tittup.v.01"},{"index":4,"sense_key":"ruffle%2:37:01::","synset_id":"ruffle.v.04"},{"index":5,"sense_key":"ruffle%2:35:03::","synset_id":"flick.v.06"},{"index":6,"sense_key":"ruffle%2:35:04::","synset_id":"shuffle.v.03"},{"index":7,"sense_key":"ruffle%2:35:01::","synset_id":"ruffle.v.07"},{"index":8,"sense_key":"ruffle%2:35:02::","synset_id":"ruffle.v.08"},{"index":9,"sense_key":"ruffle%2:35:00::","synset_id":"ruffle.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"ruffle%2:37:01::","call_ids":["senseval2.d002.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"ruffle%2:37:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008624,"input_uncached_usd":0.0000368,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.382966790988576}
{"item_id":"senseval2.d002.s080.t002","gold_sense_keys":["fact%1:10:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fact%1:10:01::","call_ids":["senseval2.d002.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fact%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039115,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.909566417045426}
{"item_id":"senseval2.d002.s080.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval2.d002.s080.t004__v1__a1","senseval2.d002.s080.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00070045,"input_uncached_usd":0.00004764999999999999,"input_cached_usd":0.0,"output_usd":0.0006528,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.432510834012646}
{"item_id":"senseval2.d002.s080.t005","gold_sense_keys":["so%4:02:02::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"so%4:02:07::","call_ids":["senseval2.d002.s080.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"so%4:02:07::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066265,"input_uncached_usd":0.00004185,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.92475949996151}
{"item_id":"senseval2.d002.s081.t000","gold_sense_keys":["publish%2:32:00::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["senseval2.d002.s081.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003614499999999999,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.398745124984998}
{"item_id":"senseval3.d000.s000.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019549999999999998,"input_uncached_usd":0.000035499999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0991415419848636}
{"item_id":"senseval3.d000.s001.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001793,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8557955409632996}
{"item_id":"senseval3.d000.s002.t000","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":2066,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00087815,"input_uncached_usd":0.00005175,"input_cached_usd":0.0,"output_usd":0.0008263999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.04872175003402}
{"item_id":"senseval3.d000.s002.t002","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval3.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.000491,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.691484250011854}
{"item_id":"senseval3.d000.s002.t004","gold_sense_keys":["surprise%1:12:00::"],"candidates":[{"index":1,"sense_key":"surprise%1:12:00::","synset_id":"surprise.n.01"},{"index":2,"sense_key":"surprise%1:11:00::","synset_id":"surprise.n.02"},{"index":3,"sense_key":"surprise%1:04:00::","synset_id":"surprise.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"surprise%1:12:00::","call_ids":["senseval3.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"surprise%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":280,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00014879999999999998,"input_uncached_usd":0.000014,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7974309169803746}
{"item_id":"senseval3.d000.s002.t005","gold_sense_keys":["fit%1:26:00::"],"candidates":[{"index":1,"sense_key":"fit%1:26:01::","synset_id":"fit.n.01"},{"index":2,"sense_key":"fit%1:26:00::","synset_id":"paroxysm.n.01"},{"index":3,"sense_key":"fit%1:07:00::","synset_id":"fit.n.03"},{"index":4,"sense_key":"fit%1:04:00::","synset_id":"burst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fit%1:26:00::","call_ids":["senseval3.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fit%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015279999999999997,"input_uncached_usd":0.0000184,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6897192500182427}
{"item_id":"senseval3.d000.s003.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004028,"input_uncached_usd":0.000037999999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.012791540997569}
{"item_id":"senseval3.d000.s003.t001","gold_sense_keys":["drunk%3:00:00::"],"candidates":[{"index":1,"sense_key":"drunk%3:00:00::","synset_id":"intoxicated.a.01"},{"index":2,"sense_key":"drunk%5:00:00:excited:00","synset_id":"intoxicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drunk%3:00:00::","call_ids":["senseval3.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drunk%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":2385,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.0009704499999999999,"input_uncached_usd":0.00001645,"input_cached_usd":0.0,"output_usd":0.000954,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.956784249981865}
{"item_id":"senseval3.d000.s003.t002","gold_sense_keys":["crazy%5:00:00:insane:00"],"candidates":[{"index":1,"sense_key":"crazy%5:00:00:insane:00","synset_id":"brainsick.s.01"},{"index":2,"sense_key":"crazy%5:00:00:impractical:00","synset_id":"crazy.s.02"},{"index":3,"sense_key":"crazy%5:00:00:excited:00","synset_id":"crazy.s.03"},{"index":4,"sense_key":"crazy%5:00:00:strange:00","synset_id":"crazy.s.04"},{"index":5,"sense_key":"crazy%5:00:00:enthusiastic:00","synset_id":"crazy.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crazy%5:00:00:insane:00","call_ids":["senseval3.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crazy%5:00:00:insane:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023469999999999999,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.235159125004429}
{"item_id":"senseval3.d000.s004.t002","gold_sense_keys":["match%2:42:01::"],"candidates":[{"index":1,"sense_key":"match%2:42:00::","synset_id":"match.v.01"},{"index":2,"sense_key":"match%2:40:00::","synset_id":"match.v.02"},{"index":3,"sense_key":"match%2:35:08::","synset_id":"match.v.03"},{"index":4,"sense_key":"match%2:42:01::","synset_id":"equal.v.02"},{"index":5,"sense_key":"match%2:30:01::","synset_id":"match.v.05"},{"index":6,"sense_key":"match%2:41:00::","synset_id":"meet.v.06"},{"index":7,"sense_key":"match%2:35:10::","synset_id":"match.v.07"},{"index":8,"sense_key":"match%2:33:00::","synset_id":"pit.v.01"},{"index":9,"sense_key":"match%2:30:00::","synset_id":"match.v.09"},{"index":10,"sense_key":"match%2:30:07::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"match%2:30:00::","call_ids":["senseval3.d000.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"match%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008088500000000001,"input_uncached_usd":0.00003445,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.846805165987462}
{"item_id":"senseval3.d000.s004.t003","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005669,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.823610791994724}
{"item_id":"senseval3.d000.s004.t004","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drink%1:04:00::","call_ids":["senseval3.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drink%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023449999999999998,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.26322074997006}
{"item_id":"senseval3.d000.s004.t005","gold_sense_keys":["lose%2:40:02::"],"candidates":[{"index":1,"sense_key":"lose%2:40:02::","synset_id":"lose.v.01"},{"index":2,"sense_key":"lose%2:33:00::","synset_id":"lose.v.02"},{"index":3,"sense_key":"lose%2:37:00::","synset_id":"lose.v.03"},{"index":4,"sense_key":"lose%2:35:00::","synset_id":"misplace.v.01"},{"index":5,"sense_key":"lose%2:40:01::","synset_id":"lose.v.05"},{"index":6,"sense_key":"lose%2:39:00::","synset_id":"lose.v.06"},{"index":7,"sense_key":"lose%2:40:06::","synset_id":"lose.v.07"},{"index":8,"sense_key":"lose%2:40:00::","synset_id":"lose.v.08"},{"index":9,"sense_key":"lose%2:33:01::","synset_id":"fall_back.v.04"},{"index":10,"sense_key":"lose%2:39:01::","synset_id":"miss.v.01"},{"index":11,"sense_key":"lose%2:30:03::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lose%2:40:02::","call_ids":["senseval3.d000.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lose%2:40:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045139999999999997,"input_uncached_usd":0.0000354,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.197256332961842}
{"item_id":"senseval3.d000.s004.t006","gold_sense_keys":["count%1:23:00::"],"candidates":[{"index":1,"sense_key":"count%1:23:00::","synset_id":"count.n.01"},{"index":2,"sense_key":"count%1:04:00::","synset_id":"count.n.02"},{"index":3,"sense_key":"count%1:18:00::","synset_id":"count.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"count%1:23:00::","call_ids":["senseval3.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"count%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":348,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035659999999999994,"input_uncached_usd":0.0000174,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.582038291962817}
{"item_id":"senseval3.d000.s004.t007","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032705,"input_uncached_usd":0.00003905,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7009037909447215}
{"item_id":"senseval3.d000.s004.t008","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.2106902079540305}
{"item_id":"senseval3.d000.s005.t000","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010309999999999999,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.111178499995731}
{"item_id":"senseval3.d000.s005.t002","gold_sense_keys":["think%2:31:01::","think%2:31:03::","think%2:31:12::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"think%2:31:13::","call_ids":["senseval3.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"think%2:31:13::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":833,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040645,"input_uncached_usd":0.000041649999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.468926667002961}
{"item_id":"senseval3.d000.s005.t004","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005414500000000001,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.974566750053782}
{"item_id":"senseval3.d000.s006.t000","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023429999999999998,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.033434540964663}
{"item_id":"senseval3.d000.s006.t002","gold_sense_keys":["bright%3:00:00::"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bright%3:00:00::","call_ids":["senseval3.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bright%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005300000000000001,"input_uncached_usd":0.000037199999999999996,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.032202500035055}
{"item_id":"senseval3.d000.s006.t003","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":2,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"},{"index":3,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strange%3:00:00::","call_ids":["senseval3.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strange%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015495,"input_uncached_usd":0.000020549999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.948700667009689}
{"item_id":"senseval3.d000.s006.t004","gold_sense_keys":["intensity%1:07:00::"],"candidates":[{"index":1,"sense_key":"intensity%1:07:03::","synset_id":"intensity.n.01"},{"index":2,"sense_key":"intensity%1:07:00::","synset_id":"intensity.n.02"},{"index":3,"sense_key":"intensity%1:07:02::","synset_id":"volume.n.06"},{"index":4,"sense_key":"intensity%1:07:01::","synset_id":"saturation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"intensity%1:07:00::","call_ids":["senseval3.d000.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"intensity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00071955,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.30447654100135}
{"item_id":"senseval3.d000.s006.t005","gold_sense_keys":["cold%3:00:02::"],"candidates":[{"index":1,"sense_key":"cold%3:00:01::","synset_id":"cold.a.01"},{"index":2,"sense_key":"cold%3:00:02::","synset_id":"cold.a.02"},{"index":3,"sense_key":"cold%5:00:00:stale:00","synset_id":"cold.s.01"},{"index":4,"sense_key":"cold%5:00:00:cool:03","synset_id":"cold.s.02"},{"index":5,"sense_key":"cold%5:00:00:perfect:00","synset_id":"cold.s.03"},{"index":6,"sense_key":"cold%5:00:00:unoriginal:00","synset_id":"cold.s.04"},{"index":7,"sense_key":"cold%5:00:00:intense:00","synset_id":"cold.s.05"},{"index":8,"sense_key":"cold%5:00:00:unloving:00","synset_id":"cold.s.06"},{"index":9,"sense_key":"cold%5:00:00:inhumane:00","synset_id":"cold.s.07"},{"index":10,"sense_key":"cold%5:00:00:unenthusiastic:00","synset_id":"cold.s.08"},{"index":11,"sense_key":"cold%5:00:00:unconscious:00","synset_id":"cold.s.09"},{"index":12,"sense_key":"cold%5:00:00:far:00","synset_id":"cold.s.10"},{"index":13,"sense_key":"cold%5:00:00:dead:01","synset_id":"cold.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"cold%5:00:00:intense:00","call_ids":["senseval3.d000.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"cold%5:00:00:intense:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002516,"input_uncached_usd":0.0000404,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.02823024999816}
{"item_id":"senseval3.d000.s006.t006","gold_sense_keys":["fire%1:12:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"fire%1:12:00::","call_ids":["senseval3.d000.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"fire%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001673,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7498436250025406}
{"item_id":"senseval3.d000.s006.t007","gold_sense_keys":["burn%2:43:01::"],"candidates":[{"index":1,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":2,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":3,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":4,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":5,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":6,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":7,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":8,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":9,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":10,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":11,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":12,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":13,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":14,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":15,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"burn%2:43:01::","call_ids":["senseval3.d000.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"burn%2:43:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056035,"input_uncached_usd":0.000041949999999999996,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.004293166042771}
{"item_id":"senseval3.d000.s007.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003364,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.494093707995489}
{"item_id":"senseval3.d000.s008.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":463,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022445,"input_uncached_usd":0.00003925,"input_cached_usd":0.0,"output_usd":0.0001852,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.9152611659956165}
{"item_id":"senseval3.d000.s008.t001","gold_sense_keys":["drunk%3:00:00::"],"candidates":[{"index":1,"sense_key":"drunk%3:00:00::","synset_id":"intoxicated.a.01"},{"index":2,"sense_key":"drunk%5:00:00:excited:00","synset_id":"intoxicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drunk%3:00:00::","call_ids":["senseval3.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drunk%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006897,"input_uncached_usd":0.0000177,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.490174957958516}
{"item_id":"senseval3.d000.s009.t000","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022999999999999998,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2582920000422746}
{"item_id":"senseval3.d000.s009.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1881,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00033085,"input_uncached_usd":0.00009405,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.758575583982747}
{"item_id":"senseval3.d000.s009.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s009.t003__v1__a1","senseval3.d000.s009.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":2336,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009677500000000001,"input_uncached_usd":0.00003335,"input_cached_usd":0.0,"output_usd":0.0009343999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.187697873974685}
{"item_id":"senseval3.d000.s010.t000","gold_sense_keys":["remember%2:31:00::"],"candidates":[{"index":1,"sense_key":"remember%2:31:00::","synset_id":"remember.v.01"},{"index":2,"sense_key":"remember%2:31:01::","synset_id":"remember.v.02"},{"index":3,"sense_key":"remember%2:31:02::","synset_id":"remember.v.03"},{"index":4,"sense_key":"remember%2:40:00::","synset_id":"remember.v.04"},{"index":5,"sense_key":"remember%2:32:07::","synset_id":"remember.v.05"},{"index":6,"sense_key":"remember%2:32:00::","synset_id":"commend.v.05"},{"index":7,"sense_key":"remember%2:31:08::","synset_id":"remember.v.07"},{"index":8,"sense_key":"remember%2:31:09::","synset_id":"commemorate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"remember%2:31:02::","call_ids":["senseval3.d000.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"remember%2:31:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011344999999999999,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.151332249981351}
{"item_id":"senseval3.d000.s011.t001","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015499999999999997,"input_uncached_usd":0.0000206,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.597440999990795}
{"item_id":"senseval3.d000.s011.t002","gold_sense_keys":["steady%3:00:00::"],"candidates":[{"index":1,"sense_key":"steady%3:00:00::","synset_id":"steady.a.01"},{"index":2,"sense_key":"steady%5:00:00:stable:00","synset_id":"firm.s.06"},{"index":3,"sense_key":"steady%5:00:00:secure:03","synset_id":"steady.s.02"},{"index":4,"sense_key":"steady%5:00:00:resolute:00","synset_id":"firm.s.01"},{"index":5,"sense_key":"steady%5:00:00:frequent:00","synset_id":"regular.s.06"},{"index":6,"sense_key":"steady%5:00:00:unexcitable:00","synset_id":"steady.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"steady%5:00:00:frequent:00","call_ids":["senseval3.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"steady%5:00:00:frequent:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003389,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.07778849999886}
{"item_id":"senseval3.d000.s011.t003","gold_sense_keys":["drinking%1:04:01::"],"candidates":[{"index":1,"sense_key":"drinking%1:04:00::","synset_id":"drinking.n.01"},{"index":2,"sense_key":"drinking%1:04:01::","synset_id":"drink.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drinking%1:04:01::","call_ids":["senseval3.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drinking%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043275,"input_uncached_usd":0.00001675,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0597912080120295}
{"item_id":"senseval3.d000.s012.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":325,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053465,"input_uncached_usd":0.00001625,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.142125999962445}
{"item_id":"senseval3.d000.s012.t001","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"head%1:09:00::","call_ids":["senseval3.d000.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"head%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1547,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00044215,"input_uncached_usd":0.00007735,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.780823124980088}
{"item_id":"senseval3.d000.s012.t003","gold_sense_keys":["thought%1:09:00::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"},{"index":3,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":4,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thought%1:09:00::","call_ids":["senseval3.d000.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thought%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041345,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0990528340335}
{"item_id":"senseval3.d000.s012.t004","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval3.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006653,"input_uncached_usd":0.0000445,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.821190499991644}
{"item_id":"senseval3.d000.s012.t005","gold_sense_keys":["cross%2:38:00::"],"candidates":[{"index":1,"sense_key":"cross%2:38:00::","synset_id":"traverse.v.01"},{"index":2,"sense_key":"cross%2:38:03::","synset_id":"intersect.v.01"},{"index":3,"sense_key":"cross%2:41:00::","synset_id":"thwart.v.01"},{"index":4,"sense_key":"cross%2:38:04::","synset_id":"cross.v.04"},{"index":5,"sense_key":"cross%2:42:00::","synset_id":"cross.v.05"},{"index":6,"sense_key":"cross%2:38:01::","synset_id":"cross.v.06"},{"index":7,"sense_key":"cross%2:36:09::","synset_id":"cross.v.07"},{"index":8,"sense_key":"cross%2:35:00::","synset_id":"crossbreed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cross%2:38:00::","call_ids":["senseval3.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cross%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044629999999999995,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.647624375007581}
{"item_id":"senseval3.d000.s012.t006","gold_sense_keys":["street%1:06:01::"],"candidates":[{"index":1,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":2,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":3,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":4,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":5,"sense_key":"street%1:14:00::","synset_id":"street.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"street%1:06:01::","call_ids":["senseval3.d000.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"street%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023519999999999997,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2179700420238078}
{"item_id":"senseval3.d000.s012.t007","gold_sense_keys":["light%1:10:00::"],"candidates":[{"index":1,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":2,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":3,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":4,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":5,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":6,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":7,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":8,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":9,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":10,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":11,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":12,"sense_key":"light%1:09:02::","synset_id":"light.n.12"},{"index":13,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":14,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":15,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"light%1:10:00::","call_ids":["senseval3.d000.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038369999999999995,"input_uncached_usd":0.0000445,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.8346086670062505}
{"item_id":"senseval3.d000.s012.t010","gold_sense_keys":["roar%2:32:01::"],"candidates":[{"index":1,"sense_key":"roar%2:32:01::","synset_id":"roar.v.01"},{"index":2,"sense_key":"roar%2:32:03::","synset_id":"thunder.v.02"},{"index":3,"sense_key":"roar%2:32:02::","synset_id":"howl.v.01"},{"index":4,"sense_key":"roar%2:38:00::","synset_id":"roar.v.04"},{"index":5,"sense_key":"roar%2:32:05::","synset_id":"bellow.v.02"},{"index":6,"sense_key":"roar%2:29:00::","synset_id":"roar.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"roar%2:32:01::","call_ids":["senseval3.d000.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"roar%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026365,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.355304917029571}
{"item_id":"senseval3.d000.s012.t011","gold_sense_keys":["intersection%1:06:00::"],"candidates":[{"index":1,"sense_key":"intersection%1:09:00::","synset_id":"intersection.n.01"},{"index":2,"sense_key":"intersection%1:06:00::","synset_id":"intersection.n.02"},{"index":3,"sense_key":"intersection%1:25:00::","synset_id":"intersection.n.03"},{"index":4,"sense_key":"intersection%1:14:00::","synset_id":"intersection.n.04"},{"index":5,"sense_key":"intersection%1:09:01::","synset_id":"overlap.n.01"},{"index":6,"sense_key":"intersection%1:04:00::","synset_id":"intersection.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"intersection%1:06:00::","call_ids":["senseval3.d000.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"intersection%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.000159,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3772002910263836}
{"item_id":"senseval3.d000.s013.t000","gold_sense_keys":["brake%1:06:00::"],"candidates":[{"index":1,"sense_key":"brake%1:06:00::","synset_id":"brake.n.01"},{"index":2,"sense_key":"brake%1:20:02::","synset_id":"brake.n.02"},{"index":3,"sense_key":"brake%1:20:01::","synset_id":"bracken.n.02"},{"index":4,"sense_key":"brake%1:14:00::","synset_id":"brake.n.04"},{"index":5,"sense_key":"brake%1:06:01::","synset_id":"brake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brake%1:06:00::","call_ids":["senseval3.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brake%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018375,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7854687919607386}
{"item_id":"senseval3.d000.s013.t001","gold_sense_keys":["howl%2:32:01::"],"candidates":[{"index":1,"sense_key":"howl%2:32:02::","synset_id":"howl.v.01"},{"index":2,"sense_key":"howl%2:32:03::","synset_id":"howl.v.02"},{"index":3,"sense_key":"howl%2:32:01::","synset_id":"roar.v.01"},{"index":4,"sense_key":"howl%2:29:00::","synset_id":"roar.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"howl%2:32:01::","call_ids":["senseval3.d000.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"howl%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015744999999999998,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2635301660047844}
{"item_id":"senseval3.d000.s013.t002","gold_sense_keys":["horn%1:06:05::"],"candidates":[{"index":1,"sense_key":"horn%1:06:06::","synset_id":"horn.n.01"},{"index":2,"sense_key":"horn%1:05:01::","synset_id":"horn.n.02"},{"index":3,"sense_key":"horn%1:10:02::","synset_id":"horn.n.03"},{"index":4,"sense_key":"horn%1:06:04::","synset_id":"horn.n.04"},{"index":5,"sense_key":"horn%1:06:01::","synset_id":"cornet.n.01"},{"index":6,"sense_key":"horn%1:05:02::","synset_id":"horn.n.06"},{"index":7,"sense_key":"horn%1:27:00::","synset_id":"horn.n.07"},{"index":8,"sense_key":"horn%1:06:07::","synset_id":"horn.n.08"},{"index":9,"sense_key":"horn%1:06:03::","synset_id":"horn.n.09"},{"index":10,"sense_key":"horn%1:06:00::","synset_id":"french_horn.n.01"},{"index":11,"sense_key":"horn%1:06:05::","synset_id":"automobile_horn.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"horn%1:10:02::","call_ids":["senseval3.d000.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"horn%1:10:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007065999999999999,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.784953458001837}
{"item_id":"senseval3.d000.s013.t003","gold_sense_keys":["blare%2:39:00::"],"candidates":[{"index":1,"sense_key":"blare%2:39:01::","synset_id":"blast.v.01"},{"index":2,"sense_key":"blare%2:39:00::","synset_id":"honk.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"blare%2:39:00::","call_ids":["senseval3.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"blare%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003317,"input_uncached_usd":0.0000181,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.707114792021457}
{"item_id":"senseval3.d000.s013.t004","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045674999999999996,"input_uncached_usd":0.00004075,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.783772249997128}
{"item_id":"senseval3.d000.s013.t005","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1016,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002876,"input_uncached_usd":0.000050799999999999995,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.253385166986845}
{"item_id":"senseval3.d000.s013.t006","gold_sense_keys":["call_out%2:32:01::"],"candidates":[{"index":1,"sense_key":"call_out%2:32:01::","synset_id":"exclaim.v.01"},{"index":2,"sense_key":"call_out%2:32:00::","synset_id":"call_out.v.02"},{"index":3,"sense_key":"call_out%2:32:12::","synset_id":"call_out.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"call_out%2:32:01::","call_ids":["senseval3.d000.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"call_out%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002822,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.6424070409848355}
{"item_id":"senseval3.d000.s013.t007","gold_sense_keys":["second%1:28:00::","second%1:28:01::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d000.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039945,"input_uncached_usd":0.000034649999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.489163791004103}
{"item_id":"senseval3.d000.s014.t002","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048594999999999997,"input_uncached_usd":0.00004435,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.869260916020721}
{"item_id":"senseval3.d000.s015.t001","gold_sense_keys":["care%2:37:02::"],"candidates":[{"index":1,"sense_key":"care%2:37:02::","synset_id":"care.v.01"},{"index":2,"sense_key":"care%2:41:13::","synset_id":"care.v.02"},{"index":3,"sense_key":"care%2:37:00::","synset_id":"wish.v.02"},{"index":4,"sense_key":"care%2:41:11::","synset_id":"manage.v.02"},{"index":5,"sense_key":"care%2:37:12::","synset_id":"worry.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"care%2:37:02::","call_ids":["senseval3.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"care%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028815,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.75721816602163}
{"item_id":"senseval3.d000.s015.t002","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015654999999999998,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2176798330037855}
{"item_id":"senseval3.d000.s015.t003","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005707500000000001,"input_uncached_usd":0.00005235,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.6657541249878705}
{"item_id":"senseval3.d000.s016.t000","gold_sense_keys":["regret%2:37:00::"],"candidates":[{"index":1,"sense_key":"regret%2:37:00::","synset_id":"repent.v.02"},{"index":2,"sense_key":"regret%2:37:01::","synset_id":"regret.v.02"},{"index":3,"sense_key":"regret%2:32:00::","synset_id":"regret.v.03"},{"index":4,"sense_key":"regret%2:32:02::","synset_id":"regret.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"regret%2:37:00::","call_ids":["senseval3.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"regret%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023434999999999998,"input_uncached_usd":0.00002315,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.959038667031564}
{"item_id":"senseval3.d000.s016.t001","gold_sense_keys":["open%2:35:00::"],"candidates":[{"index":1,"sense_key":"open%2:35:00::","synset_id":"open.v.01"},{"index":2,"sense_key":"open%2:41:00::","synset_id":"open.v.02"},{"index":3,"sense_key":"open%2:35:06::","synset_id":"open.v.03"},{"index":4,"sense_key":"open%2:41:01::","synset_id":"open.v.04"},{"index":5,"sense_key":"open%2:35:08::","synset_id":"unfold.v.04"},{"index":6,"sense_key":"open%2:30:00::","synset_id":"open.v.06"},{"index":7,"sense_key":"open%2:30:01::","synset_id":"open.v.07"},{"index":8,"sense_key":"open%2:42:00::","synset_id":"open.v.08"},{"index":9,"sense_key":"open%2:33:00::","synset_id":"open.v.09"},{"index":10,"sense_key":"open%2:40:04::","synset_id":"afford.v.04"},{"index":11,"sense_key":"open%2:39:10::","synset_id":"open.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"open%2:35:00::","call_ids":["senseval3.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"open%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":1170,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050365,"input_uncached_usd":0.00003565,"input_cached_usd":0.0,"output_usd":0.000468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.12708887498593}
{"item_id":"senseval3.d000.s016.t002","gold_sense_keys":["mouth%1:08:01::"],"candidates":[{"index":1,"sense_key":"mouth%1:08:01::","synset_id":"mouth.n.01"},{"index":2,"sense_key":"mouth%1:08:00::","synset_id":"mouth.n.02"},{"index":3,"sense_key":"mouth%1:17:01::","synset_id":"mouth.n.03"},{"index":4,"sense_key":"mouth%1:17:00::","synset_id":"mouth.n.04"},{"index":5,"sense_key":"mouth%1:18:00::","synset_id":"mouth.n.05"},{"index":6,"sense_key":"mouth%1:18:01::","synset_id":"mouthpiece.n.03"},{"index":7,"sense_key":"mouth%1:10:00::","synset_id":"sass.n.01"},{"index":8,"sense_key":"mouth%1:06:00::","synset_id":"mouth.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mouth%1:08:01::","call_ids":["senseval3.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mouth%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039825,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.984975499974098}
{"item_id":"senseval3.d000.s016.t003","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s016.t003__v1__a1","senseval3.d000.s016.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":865,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041444999999999996,"input_uncached_usd":0.000043249999999999994,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.478386875009164}
{"item_id":"senseval3.d000.s016.t005","gold_sense_keys":["stop%1:11:00::"],"candidates":[{"index":1,"sense_key":"stop%1:11:00::","synset_id":"stop.n.01"},{"index":2,"sense_key":"stop%1:04:00::","synset_id":"stop.n.02"},{"index":3,"sense_key":"stop%1:04:02::","synset_id":"stop.n.03"},{"index":4,"sense_key":"stop%1:26:00::","synset_id":"arrest.n.02"},{"index":5,"sense_key":"stop%1:15:00::","synset_id":"stop.n.05"},{"index":6,"sense_key":"stop%1:10:00::","synset_id":"stop_consonant.n.01"},{"index":7,"sense_key":"stop%1:10:01::","synset_id":"period.n.07"},{"index":8,"sense_key":"stop%1:06:03::","synset_id":"stop.n.08"},{"index":9,"sense_key":"stop%1:06:02::","synset_id":"diaphragm.n.01"},{"index":10,"sense_key":"stop%1:06:00::","synset_id":"catch.n.06"},{"index":11,"sense_key":"stop%1:06:01::","synset_id":"blockage.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stop%1:11:00::","call_ids":["senseval3.d000.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stop%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004594,"input_uncached_usd":0.0000434,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.634577207965776}
{"item_id":"senseval3.d000.s016.t006","gold_sense_keys":["angry%3:00:00::"],"candidates":[{"index":1,"sense_key":"angry%3:00:00::","synset_id":"angry.a.01"},{"index":2,"sense_key":"angry%5:00:00:stormy:00","synset_id":"angry.s.01"},{"index":3,"sense_key":"angry%5:00:00:unhealthy:00","synset_id":"angry.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"angry%3:00:00::","call_ids":["senseval3.d000.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"angry%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013145,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1243104170425795}
{"item_id":"senseval3.d000.s016.t007","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":2,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":3,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"},{"index":4,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":5,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015854999999999998,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1474619589862414}
{"item_id":"senseval3.d000.s016.t008","gold_sense_keys":["jump%2:38:11::"],"candidates":[{"index":1,"sense_key":"jump%2:38:00::","synset_id":"jump.v.01"},{"index":2,"sense_key":"jump%2:38:04::","synset_id":"startle.v.02"},{"index":3,"sense_key":"jump%2:33:00::","synset_id":"jump.v.03"},{"index":4,"sense_key":"jump%2:30:00::","synset_id":"jump.v.04"},{"index":5,"sense_key":"jump%2:42:00::","synset_id":"leap_out.v.01"},{"index":6,"sense_key":"jump%2:33:03::","synset_id":"jump.v.06"},{"index":7,"sense_key":"jump%2:30:01::","synset_id":"rise.v.11"},{"index":8,"sense_key":"jump%2:38:11::","synset_id":"jump.v.08"},{"index":9,"sense_key":"jump%2:38:05::","synset_id":"derail.v.02"},{"index":10,"sense_key":"jump%2:38:12::","synset_id":"chute.v.01"},{"index":11,"sense_key":"jump%2:38:01::","synset_id":"jump.v.11"},{"index":12,"sense_key":"jump%2:38:07::","synset_id":"jumpstart.v.01"},{"index":13,"sense_key":"jump%2:31:00::","synset_id":"jump.v.13"},{"index":14,"sense_key":"jump%2:30:05::","synset_id":"leap.v.02"},{"index":15,"sense_key":"jump%2:30:03::","synset_id":"alternate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"jump%2:38:11::","call_ids":["senseval3.d000.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"jump%2:38:11::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002319,"input_uncached_usd":0.0000463,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2366474580485374}
{"item_id":"senseval3.d000.s016.t009","gold_sense_keys":["cab%1:06:02::"],"candidates":[{"index":1,"sense_key":"cab%1:06:02::","synset_id":"cab.n.01"},{"index":2,"sense_key":"cab%1:06:01::","synset_id":"cab.n.02"},{"index":3,"sense_key":"cab%1:06:00::","synset_id":"cab.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cab%1:06:02::","call_ids":["senseval3.d000.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cab%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015659999999999998,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8348507920163684}
{"item_id":"senseval3.d000.s016.t010","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"walk%2:38:03::","call_ids":["senseval3.d000.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"walk%2:38:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047509999999999995,"input_uncached_usd":0.0000335,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.263426374993287}
{"item_id":"senseval3.d000.s017.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015419999999999998,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2892602090141736}
{"item_id":"senseval3.d000.s017.t003","gold_sense_keys":["arm%1:08:00::"],"candidates":[{"index":1,"sense_key":"arm%1:08:00::","synset_id":"arm.n.01"},{"index":2,"sense_key":"arm%1:06:03::","synset_id":"arm.n.02"},{"index":3,"sense_key":"arm%1:06:01::","synset_id":"weapon.n.01"},{"index":4,"sense_key":"arm%1:06:02::","synset_id":"arm.n.04"},{"index":5,"sense_key":"arm%1:14:00::","synset_id":"branch.n.01"},{"index":6,"sense_key":"arm%1:06:00::","synset_id":"sleeve.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arm%1:08:00::","call_ids":["senseval3.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arm%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021519999999999997,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3469264159793966}
{"item_id":"senseval3.d000.s017.t004","gold_sense_keys":["shoulder%1:08:00::"],"candidates":[{"index":1,"sense_key":"shoulder%1:08:00::","synset_id":"shoulder.n.01"},{"index":2,"sense_key":"shoulder%1:13:00::","synset_id":"shoulder.n.02"},{"index":3,"sense_key":"shoulder%1:08:02::","synset_id":"shoulder.n.03"},{"index":4,"sense_key":"shoulder%1:06:01::","synset_id":"shoulder.n.04"},{"index":5,"sense_key":"shoulder%1:06:00::","synset_id":"shoulder.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shoulder%1:08:00::","call_ids":["senseval3.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shoulder%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016104999999999998,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9109651669859886}
{"item_id":"senseval3.d000.s017.t005","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"old%5:00:00:familiar:00","call_ids":["senseval3.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"old%5:00:00:familiar:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019534999999999998,"input_uncached_usd":0.00003535,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9549420420080423}
{"item_id":"senseval3.d000.s017.t006","gold_sense_keys":["friend%1:18:00::"],"candidates":[{"index":1,"sense_key":"friend%1:18:00::","synset_id":"friend.n.01"},{"index":2,"sense_key":"friend%1:18:03::","synset_id":"ally.n.02"},{"index":3,"sense_key":"friend%1:18:02::","synset_id":"acquaintance.n.03"},{"index":4,"sense_key":"friend%1:18:04::","synset_id":"supporter.n.01"},{"index":5,"sense_key":"friend%1:18:01::","synset_id":"friend.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"friend%1:18:00::","call_ids":["senseval3.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"friend%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016235,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.591629166039638}
{"item_id":"senseval3.d000.s018.t000","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":2,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":3,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"},{"index":4,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":5,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"driver%1:18:01::","call_ids":["senseval3.d000.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"driver%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041475,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.981644832994789}
{"item_id":"senseval3.d000.s018.t001","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"start%2:30:00::","call_ids":["senseval3.d000.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"start%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038674999999999994,"input_uncached_usd":0.00004755,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.643733999982942}
{"item_id":"senseval3.d000.s018.t002","gold_sense_keys":["curse%2:32:02::"],"candidates":[{"index":1,"sense_key":"curse%2:32:00::","synset_id":"curse.v.01"},{"index":2,"sense_key":"curse%2:32:02::","synset_id":"curse.v.02"},{"index":3,"sense_key":"curse%2:32:01::","synset_id":"curse.v.03"},{"index":4,"sense_key":"curse%2:32:03::","synset_id":"excommunicate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"curse%2:32:00::","call_ids":["senseval3.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"curse%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005188,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.7309026250150055}
{"item_id":"senseval3.d000.s018.t003","gold_sense_keys":["plot%1:09:00::"],"candidates":[{"index":1,"sense_key":"plot%1:09:00::","synset_id":"plot.n.01"},{"index":2,"sense_key":"plot%1:15:00::","synset_id":"plot.n.02"},{"index":3,"sense_key":"plot%1:10:00::","synset_id":"plot.n.03"},{"index":4,"sense_key":"plot%1:10:01::","synset_id":"plot.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"plot%1:09:00::","call_ids":["senseval3.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"plot%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015994999999999998,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9063295000232756}
{"item_id":"senseval3.d000.s018.t004","gold_sense_keys":["ruin%2:41:00::"],"candidates":[{"index":1,"sense_key":"ruin%2:35:00::","synset_id":"destroy.v.02"},{"index":2,"sense_key":"ruin%2:41:00::","synset_id":"ruin.v.02"},{"index":3,"sense_key":"ruin%2:40:00::","synset_id":"bankrupt.v.01"},{"index":4,"sense_key":"ruin%2:35:01::","synset_id":"ruin.v.04"},{"index":5,"sense_key":"ruin%2:35:02::","synset_id":"deflower.v.01"},{"index":6,"sense_key":"ruin%2:30:00::","synset_id":"ruin.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ruin%2:41:00::","call_ids":["senseval3.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ruin%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018805,"input_uncached_usd":0.000028049999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0796107909991406}
{"item_id":"senseval3.d000.s018.t006","gold_sense_keys":["record%1:04:01::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"record%1:04:01::","call_ids":["senseval3.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"record%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027374999999999996,"input_uncached_usd":0.00003695,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.5772472500102594}
{"item_id":"senseval3.d000.s019.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035624999999999996,"input_uncached_usd":0.00004265,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7702309169690125}
{"item_id":"senseval3.d000.s019.t001","gold_sense_keys":["save%2:41:00::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"save%2:41:00::","call_ids":["senseval3.d000.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"save%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":2129,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.0008884499999999998,"input_uncached_usd":0.00003685,"input_cached_usd":0.0,"output_usd":0.0008516,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.923583083960693}
{"item_id":"senseval3.d000.s019.t002","gold_sense_keys":["turn%2:38:00::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"turn%2:38:00::","call_ids":["senseval3.d000.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"turn%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1440,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.000488,"input_uncached_usd":0.000072,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.635509165993426}
{"item_id":"senseval3.d000.s019.t003","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001724,"input_uncached_usd":0.000037999999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.901372166001238}
{"item_id":"senseval3.d000.s019.t005","gold_sense_keys":["face%1:08:00::"],"candidates":[{"index":1,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":2,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":3,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":4,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":5,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":6,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":7,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":8,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":9,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":10,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":11,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":12,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":13,"sense_key":"face%1:06:00::","synset_id":"face.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%1:08:00::","call_ids":["senseval3.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025289999999999997,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.670473041012883}
{"item_id":"senseval3.d000.s019.t006","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030234999999999995,"input_uncached_usd":0.000039949999999999995,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.854871332994662}
{"item_id":"senseval3.d000.s019.t007","gold_sense_keys":["word%1:10:03::"],"candidates":[{"index":1,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":2,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":3,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":4,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":5,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":6,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":7,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":8,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":9,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":10,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"word%1:10:03::","call_ids":["senseval3.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"word%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001456,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.0001092,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3014208750100806}
{"item_id":"senseval3.d000.s020.t000","gold_sense_keys":["driver%1:18:00::"],"candidates":[{"index":1,"sense_key":"driver%1:18:00::","synset_id":"driver.n.01"},{"index":2,"sense_key":"driver%1:18:02::","synset_id":"driver.n.02"},{"index":3,"sense_key":"driver%1:18:01::","synset_id":"driver.n.03"},{"index":4,"sense_key":"driver%1:10:00::","synset_id":"driver.n.04"},{"index":5,"sense_key":"driver%1:06:00::","synset_id":"driver.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"driver%1:18:00::","call_ids":["senseval3.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"driver%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046459999999999996,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.924222250003368}
{"item_id":"senseval3.d000.s020.t001","gold_sense_keys":["stop%2:42:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stop%2:42:00::","call_ids":["senseval3.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035479999999999995,"input_uncached_usd":0.0000412,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.228818541974761}
{"item_id":"senseval3.d000.s020.t002","gold_sense_keys":["swear%2:32:01::"],"candidates":[{"index":1,"sense_key":"swear%2:32:01::","synset_id":"curse.v.01"},{"index":2,"sense_key":"swear%2:32:03::","synset_id":"affirm.v.02"},{"index":3,"sense_key":"swear%2:32:02::","synset_id":"swear.v.03"},{"index":4,"sense_key":"swear%2:32:00::","synset_id":"swear.v.04"},{"index":5,"sense_key":"swear%2:31:10::","synset_id":"trust.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swear%2:32:01::","call_ids":["senseval3.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swear%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031385,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.147461874992587}
{"item_id":"senseval3.d000.s020.t003","gold_sense_keys":["turn%2:38:00::"],"candidates":[{"index":1,"sense_key":"turn%2:38:00::","synset_id":"turn.v.01"},{"index":2,"sense_key":"turn%2:30:04::","synset_id":"change_state.v.01"},{"index":3,"sense_key":"turn%2:42:00::","synset_id":"become.v.02"},{"index":4,"sense_key":"turn%2:38:02::","synset_id":"turn.v.04"},{"index":5,"sense_key":"turn%2:30:00::","synset_id":"change_by_reversal.v.01"},{"index":6,"sense_key":"turn%2:38:03::","synset_id":"turn.v.06"},{"index":7,"sense_key":"turn%2:30:03::","synset_id":"turn.v.07"},{"index":8,"sense_key":"turn%2:35:08::","synset_id":"turn.v.08"},{"index":9,"sense_key":"turn%2:38:04::","synset_id":"turn.v.09"},{"index":10,"sense_key":"turn%2:38:01::","synset_id":"turn.v.10"},{"index":11,"sense_key":"turn%2:38:06::","synset_id":"turn.v.11"},{"index":12,"sense_key":"turn%2:36:00::","synset_id":"plow.v.01"},{"index":13,"sense_key":"turn%2:35:06::","synset_id":"turn.v.13"},{"index":14,"sense_key":"turn%2:30:09::","synset_id":"turn.v.14"},{"index":15,"sense_key":"turn%2:29:00::","synset_id":"twist.v.10"},{"index":16,"sense_key":"turn%2:42:13::","synset_id":"turn.v.16"},{"index":17,"sense_key":"turn%2:41:10::","synset_id":"turn.v.17"},{"index":18,"sense_key":"turn%2:40:13::","synset_id":"turn.v.18"},{"index":19,"sense_key":"turn%2:38:13::","synset_id":"turn.v.19"},{"index":20,"sense_key":"turn%2:38:14::","synset_id":"turn.v.20"},{"index":21,"sense_key":"turn%2:35:11::","synset_id":"flex.v.05"},{"index":22,"sense_key":"turn%2:35:10::","synset_id":"turn.v.22"},{"index":23,"sense_key":"turn%2:33:13::","synset_id":"turn.v.23"},{"index":24,"sense_key":"turn%2:32:14::","synset_id":"call_on.v.01"},{"index":25,"sense_key":"turn%2:30:01::","synset_id":"sour.v.01"},{"index":26,"sense_key":"turn%2:30:14::","synset_id":"turn.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"turn%2:38:00::","call_ids":["senseval3.d000.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"turn%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1429,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0006158499999999999,"input_uncached_usd":0.00007145,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.739950707997195}
{"item_id":"senseval3.d000.s020.t004","gold_sense_keys":["heel%1:08:00::"],"candidates":[{"index":1,"sense_key":"heel%1:06:00::","synset_id":"heel.n.01"},{"index":2,"sense_key":"heel%1:08:00::","synset_id":"heel.n.02"},{"index":3,"sense_key":"heel%1:18:00::","synset_id":"cad.n.01"},{"index":4,"sense_key":"heel%1:13:00::","synset_id":"heel.n.04"},{"index":5,"sense_key":"heel%1:06:03::","synset_id":"heel.n.05"},{"index":6,"sense_key":"heel%1:06:02::","synset_id":"heel.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"heel%1:08:00::","call_ids":["senseval3.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"heel%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018525,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9953868750017136}
{"item_id":"senseval3.d000.s020.t005","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033285,"input_uncached_usd":0.00001925,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4440205420251}
{"item_id":"senseval3.d000.s021.t000","gold_sense_keys":["give%2:31:02::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"give%2:31:02::","call_ids":["senseval3.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"give%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2059,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00023735,"input_uncached_usd":0.00010295,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9609452089644037}
{"item_id":"senseval3.d000.s021.t002","gold_sense_keys":["thought%1:09:00::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"},{"index":3,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":4,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thought%1:09:00::","call_ids":["senseval3.d000.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thought%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031345,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.230194042029325}
{"item_id":"senseval3.d000.s022.t000","gold_sense_keys":["recall%2:31:00::"],"candidates":[{"index":1,"sense_key":"recall%2:31:00::","synset_id":"remember.v.01"},{"index":2,"sense_key":"recall%2:32:00::","synset_id":"hark_back.v.01"},{"index":3,"sense_key":"recall%2:42:00::","synset_id":"echo.v.03"},{"index":4,"sense_key":"recall%2:32:02::","synset_id":"recall.v.04"},{"index":5,"sense_key":"recall%2:31:06::","synset_id":"recall.v.05"},{"index":6,"sense_key":"recall%2:41:00::","synset_id":"recall.v.06"},{"index":7,"sense_key":"recall%2:40:00::","synset_id":"recall.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recall%2:31:00::","call_ids":["senseval3.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recall%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037099999999999996,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.655780040950049}
{"item_id":"senseval3.d000.s022.t001","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025804999999999996,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.478712000011001}
{"item_id":"senseval3.d000.s022.t002","gold_sense_keys":["see%2:36:00::","see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1441,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00043684999999999997,"input_uncached_usd":0.00007205,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.250549208954908}
{"item_id":"senseval3.d000.s022.t003","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"eye%1:09:01::","call_ids":["senseval3.d000.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"eye%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026149999999999996,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.6307250410318375}
{"item_id":"senseval3.d000.s022.t004","gold_sense_keys":["make%2:32:00::","make%2:36:15::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"make%2:36:15::","call_ids":["senseval3.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"make%2:36:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2254,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00027269999999999996,"input_uncached_usd":0.00011269999999999999,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7611989169963636}
{"item_id":"senseval3.d000.s023.t000","gold_sense_keys":["sort%1:09:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:09:00::","call_ids":["senseval3.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031125,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.069918374996632}
{"item_id":"senseval3.d000.s023.t001","gold_sense_keys":["look%1:07:01::"],"candidates":[{"index":1,"sense_key":"look%1:07:01::","synset_id":"expression.n.01"},{"index":2,"sense_key":"look%1:04:00::","synset_id":"look.n.02"},{"index":3,"sense_key":"look%1:07:00::","synset_id":"look.n.03"},{"index":4,"sense_key":"look%1:26:00::","synset_id":"spirit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%1:07:01::","call_ids":["senseval3.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015854999999999998,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9427597919711843}
{"item_id":"senseval3.d000.s023.t003","gold_sense_keys":["bluff%1:10:00::"],"candidates":[{"index":1,"sense_key":"bluff%1:17:00::","synset_id":"bluff.n.01"},{"index":2,"sense_key":"bluff%1:10:00::","synset_id":"bluff.n.02"},{"index":3,"sense_key":"bluff%1:04:00::","synset_id":"bluff.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bluff%1:10:00::","call_ids":["senseval3.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bluff%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005126000000000001,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.7622747080167755}
{"item_id":"senseval3.d000.s023.t004","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050555,"input_uncached_usd":0.00003835,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.026882666978054}
{"item_id":"senseval3.d000.s023.t005","gold_sense_keys":["word%1:10:03::"],"candidates":[{"index":1,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":2,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":3,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":4,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":5,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":6,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":7,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":8,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":9,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":10,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"word%1:10:00::","call_ids":["senseval3.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"word%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037399999999999993,"input_uncached_usd":0.0000348,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.73274120897986}
{"item_id":"senseval3.d000.s024.t000","gold_sense_keys":["light%1:10:00::"],"candidates":[{"index":1,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":2,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":3,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":4,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":5,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":6,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":7,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":8,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":9,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":10,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":11,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":12,"sense_key":"light%1:09:02::","synset_id":"light.n.12"},{"index":13,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":14,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":15,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"light%1:10:00::","call_ids":["senseval3.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007946,"input_uncached_usd":0.000045799999999999995,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.18320125003811}
{"item_id":"senseval3.d000.s024.t002","gold_sense_keys":["way%1:15:00::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"way%1:15:00::","call_ids":["senseval3.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"way%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004795499999999999,"input_uncached_usd":0.00003795,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.125472665997222}
{"item_id":"senseval3.d000.s024.t003","gold_sense_keys":["go_on%2:38:00::"],"candidates":[{"index":1,"sense_key":"go_on%2:42:00::","synset_id":"continue.v.01"},{"index":2,"sense_key":"go_on%2:30:00::","synset_id":"happen.v.01"},{"index":3,"sense_key":"go_on%2:38:00::","synset_id":"advance.v.01"},{"index":4,"sense_key":"go_on%2:32:00::","synset_id":"continue.v.02"},{"index":5,"sense_key":"go_on%2:35:00::","synset_id":"go_on.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"go_on%2:38:00::","call_ids":["senseval3.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"go_on%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028924999999999996,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.828939958999399}
{"item_id":"senseval3.d000.s024.t004","gold_sense_keys":["street%1:06:01::"],"candidates":[{"index":1,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":2,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":3,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":4,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":5,"sense_key":"street%1:14:00::","synset_id":"street.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"street%1:06:01::","call_ids":["senseval3.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"street%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013404999999999998,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4878568340209313}
{"item_id":"senseval3.d000.s025.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stranger%1:18:00::","call_ids":["senseval3.d000.s025.t000__v1__a1","senseval3.d000.s025.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stranger%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":2336,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00097225,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.0009343999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.31830500002252}
{"item_id":"senseval3.d000.s025.t001","gold_sense_keys":["find_out%2:31:00::"],"candidates":[{"index":1,"sense_key":"find_out%2:32:00::","synset_id":"determine.v.01"},{"index":2,"sense_key":"find_out%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"find_out%2:32:01::","synset_id":"determine.v.08"},{"index":4,"sense_key":"find_out%2:31:01::","synset_id":"find_out.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"find_out%2:32:01::","call_ids":["senseval3.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"find_out%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":1873,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007761499999999999,"input_uncached_usd":0.000026949999999999998,"input_cached_usd":0.0,"output_usd":0.0007492,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.680076542019378}
{"item_id":"senseval3.d000.s025.t003","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035814999999999995,"input_uncached_usd":0.00001895,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.416788166970946}
{"item_id":"senseval3.d000.s025.t004","gold_sense_keys":["bar%1:06:04::"],"candidates":[{"index":1,"sense_key":"bar%1:06:04::","synset_id":"barroom.n.01"},{"index":2,"sense_key":"bar%1:06:05::","synset_id":"bar.n.02"},{"index":3,"sense_key":"bar%1:06:00::","synset_id":"bar.n.03"},{"index":4,"sense_key":"bar%1:10:00::","synset_id":"measure.n.07"},{"index":5,"sense_key":"bar%1:06:02::","synset_id":"bar.n.05"},{"index":6,"sense_key":"bar%1:04:00::","synset_id":"prevention.n.01"},{"index":7,"sense_key":"bar%1:23:00::","synset_id":"bar.n.07"},{"index":8,"sense_key":"bar%1:17:00::","synset_id":"bar.n.08"},{"index":9,"sense_key":"bar%1:14:00::","synset_id":"legal_profession.n.01"},{"index":10,"sense_key":"bar%1:07:00::","synset_id":"stripe.n.05"},{"index":11,"sense_key":"bar%1:06:06::","synset_id":"cake.n.01"},{"index":12,"sense_key":"bar%1:06:01::","synset_id":"browning_automatic_rifle.n.01"},{"index":13,"sense_key":"bar%1:06:09::","synset_id":"bar.n.13"},{"index":14,"sense_key":"bar%1:06:08::","synset_id":"bar.n.14"},{"index":15,"sense_key":"bar%1:06:07::","synset_id":"bar.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bar%1:06:04::","call_ids":["senseval3.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bar%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":978,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00023449999999999998,"input_uncached_usd":0.000048899999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.207192166999448}
{"item_id":"senseval3.d000.s025.t005","gold_sense_keys":["insist%2:32:00::"],"candidates":[{"index":1,"sense_key":"insist%2:32:00::","synset_id":"insist.v.01"},{"index":2,"sense_key":"insist%2:32:03::","synset_id":"importune.v.01"},{"index":3,"sense_key":"insist%2:31:00::","synset_id":"insist.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"insist%2:32:00::","call_ids":["senseval3.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"insist%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003084,"input_uncached_usd":0.000020399999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.082996999961324}
{"item_id":"senseval3.d000.s025.t006","gold_sense_keys":["offer%2:41:00::"],"candidates":[{"index":1,"sense_key":"offer%2:40:02::","synset_id":"offer.v.01"},{"index":2,"sense_key":"offer%2:40:00::","synset_id":"offer.v.02"},{"index":3,"sense_key":"offer%2:41:00::","synset_id":"volunteer.v.02"},{"index":4,"sense_key":"offer%2:31:00::","synset_id":"offer.v.04"},{"index":5,"sense_key":"offer%2:40:09::","synset_id":"offer.v.05"},{"index":6,"sense_key":"offer%2:40:03::","synset_id":"offer.v.06"},{"index":7,"sense_key":"offer%2:40:01::","synset_id":"offer.v.07"},{"index":8,"sense_key":"offer%2:39:06::","synset_id":"offer.v.08"},{"index":9,"sense_key":"offer%2:42:00::","synset_id":"offer.v.09"},{"index":10,"sense_key":"offer%2:41:03::","synset_id":"put_up.v.02"},{"index":11,"sense_key":"offer%2:40:10::","synset_id":"extend.v.04"},{"index":12,"sense_key":"offer%2:32:12::","synset_id":"propose.v.05"},{"index":13,"sense_key":"offer%2:32:06::","synset_id":"offer.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"offer%2:41:00::","call_ids":["senseval3.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"offer%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003285,"input_uncached_usd":0.000040499999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.782690834021196}
{"item_id":"senseval3.d000.s025.t007","gold_sense_keys":["buy%2:40:00::"],"candidates":[{"index":1,"sense_key":"buy%2:40:00::","synset_id":"buy.v.01"},{"index":2,"sense_key":"buy%2:40:02::","synset_id":"bribe.v.01"},{"index":3,"sense_key":"buy%2:42:00::","synset_id":"buy.v.03"},{"index":4,"sense_key":"buy%2:40:03::","synset_id":"buy.v.04"},{"index":5,"sense_key":"buy%2:31:09::","synset_id":"buy.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"buy%2:40:00::","call_ids":["senseval3.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"buy%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018475,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.383749417029321}
{"item_id":"senseval3.d000.s025.t008","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drink%1:13:00::","call_ids":["senseval3.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drink%1:13:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021099999999999998,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.5473053340101615}
{"item_id":"senseval3.d000.s026.t000","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1691,"cached_input_tokens":0,"output_tokens":1681,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007569499999999998,"input_uncached_usd":0.00008455,"input_cached_usd":0.0,"output_usd":0.0006724,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.6157236250001}
{"item_id":"senseval3.d000.s026.t001","gold_sense_keys":["keep_to_oneself%2:41:00::"],"candidates":[{"index":1,"sense_key":"keep_to_oneself%2:41:00::","synset_id":"keep_to_oneself.v.01"},{"index":2,"sense_key":"keep_to_oneself%2:40:00::","synset_id":"keep_to_oneself.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"keep_to_oneself%2:41:00::","call_ids":["senseval3.d000.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"keep_to_oneself%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005114000000000001,"input_uncached_usd":0.000018599999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.070330957998522}
{"item_id":"senseval3.d000.s026.t002","gold_sense_keys":["sit%2:35:00::","sit%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":3,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":4,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":5,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":6,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":7,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":8,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":9,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":10,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027079999999999997,"input_uncached_usd":0.000034,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.100248208967969}
{"item_id":"senseval3.d000.s026.t003","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002714,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7910254579619505}
{"item_id":"senseval3.d000.s026.t004","gold_sense_keys":["pass%2:42:00::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"pass%2:42:00::","call_ids":["senseval3.d000.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"pass%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1432,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00025719999999999996,"input_uncached_usd":0.00007159999999999999,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4453330420074053}
{"item_id":"senseval3.d000.s026.t005","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval3.d000.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002205,"input_uncached_usd":0.0000349,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7653945830534212}
{"item_id":"senseval3.d000.s026.t006","gold_sense_keys":["nurse%2:37:00::"],"candidates":[{"index":1,"sense_key":"nurse%2:29:00::","synset_id":"nurse.v.01"},{"index":2,"sense_key":"nurse%2:37:00::","synset_id":"harbor.v.01"},{"index":3,"sense_key":"nurse%2:41:00::","synset_id":"nurse.v.03"},{"index":4,"sense_key":"nurse%2:41:03::","synset_id":"nurse.v.04"},{"index":5,"sense_key":"nurse%2:34:00::","synset_id":"breastfeed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nurse%2:37:00::","call_ids":["senseval3.d000.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nurse%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039119999999999997,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.8130590000073425}
{"item_id":"senseval3.d000.s026.t007","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s026.t007__v1__a1","senseval3.d000.s026.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":2592,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010771499999999998,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.0010368,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.486968957993668}
{"item_id":"senseval3.d000.s027.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d000.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023404999999999997,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0542153749847785}
{"item_id":"senseval3.d000.s027.t001","gold_sense_keys":["mind%2:32:00::"],"candidates":[{"index":1,"sense_key":"mind%2:32:00::","synset_id":"mind.v.01"},{"index":2,"sense_key":"mind%2:31:02::","synset_id":"mind.v.02"},{"index":3,"sense_key":"mind%2:41:01::","synset_id":"take_care.v.02"},{"index":4,"sense_key":"mind%2:41:00::","synset_id":"heed.v.01"},{"index":5,"sense_key":"mind%2:31:03::","synset_id":"beware.v.01"},{"index":6,"sense_key":"mind%2:31:00::","synset_id":"mind.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mind%2:32:00::","call_ids":["senseval3.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mind%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036434999999999994,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.27300375001505}
{"item_id":"senseval3.d000.s027.t002","gold_sense_keys":["company%1:26:00::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"company%1:26:00::","call_ids":["senseval3.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"company%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021754999999999998,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5061393749783747}
{"item_id":"senseval3.d000.s027.t004","gold_sense_keys":["free%5:00:00:unpaid:00"],"candidates":[{"index":1,"sense_key":"free%3:00:00::","synset_id":"free.a.01"},{"index":2,"sense_key":"free%3:00:02::","synset_id":"free.a.02"},{"index":3,"sense_key":"free%5:00:00:unpaid:00","synset_id":"complimentary.s.01"},{"index":4,"sense_key":"free%5:00:02:unoccupied:00","synset_id":"free.s.02"},{"index":5,"sense_key":"free%5:00:00:unfixed:00","synset_id":"detached.s.05"},{"index":6,"sense_key":"free%3:00:01::","synset_id":"free.a.06"},{"index":7,"sense_key":"free%5:00:01:unoccupied:00","synset_id":"spare.s.03"},{"index":8,"sense_key":"free%5:00:00:nonexistent:00","synset_id":"barren.s.03"},{"index":9,"sense_key":"free%5:00:00:inexact:00","synset_id":"free.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"free%5:00:00:unpaid:00","call_ids":["senseval3.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"free%5:00:00:unpaid:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000916,"input_uncached_usd":0.000034,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.5982099999673665}
{"item_id":"senseval3.d000.s027.t005","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drink%1:13:00::","call_ids":["senseval3.d000.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drink%1:13:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021069999999999997,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9417758749914356}
{"item_id":"senseval3.d000.s027.t006","gold_sense_keys":["make%2:40:01::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"make%2:40:01::","call_ids":["senseval3.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"make%2:40:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2244,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0002466,"input_uncached_usd":0.00011219999999999999,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8101912919664755}
{"item_id":"senseval3.d000.s027.t009","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"job%1:04:00::","call_ids":["senseval3.d000.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"job%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017614999999999997,"input_uncached_usd":0.00004175,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.853878458030522}
{"item_id":"senseval3.d000.s029.t000","gold_sense_keys":["wonder%2:32:01::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wonder%2:32:01::","call_ids":["senseval3.d000.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wonder%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.000129,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9957002499722876}
{"item_id":"senseval3.d000.s029.t002","gold_sense_keys":["screwball%1:18:00::"],"candidates":[{"index":1,"sense_key":"screwball%1:18:00::","synset_id":"crackpot.n.01"},{"index":2,"sense_key":"screwball%1:04:00::","synset_id":"screwball.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"screwball%1:18:00::","call_ids":["senseval3.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"screwball%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015194999999999998,"input_uncached_usd":0.00001755,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1618368749623187}
{"item_id":"senseval3.d000.s030.t000","gold_sense_keys":["take%2:31:07::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"take%2:31:07::","call_ids":["senseval3.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"take%2:31:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2172,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008830000000000001,"input_uncached_usd":0.0001086,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.47152558300877}
{"item_id":"senseval3.d000.s030.t001","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"offer%1:10:00::","call_ids":["senseval3.d000.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041215,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.228691124997567}
{"item_id":"senseval3.d000.s030.t003","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004591,"input_uncached_usd":0.0000427,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.434553708997555}
{"item_id":"senseval3.d000.s030.t004","gold_sense_keys":["uneasy%3:00:00::"],"candidates":[{"index":1,"sense_key":"uneasy%3:00:00::","synset_id":"uneasy.a.01"},{"index":2,"sense_key":"uneasy%3:00:05::","synset_id":"restless.a.03"},{"index":3,"sense_key":"uneasy%5:00:00:troubled:00","synset_id":"anxious.s.02"},{"index":4,"sense_key":"uneasy%5:00:00:uncomfortable:01","synset_id":"awkward.s.05"},{"index":5,"sense_key":"uneasy%5:00:00:uncomfortable:00","synset_id":"uneasy.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"uneasy%5:00:00:uncomfortable:01","call_ids":["senseval3.d000.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"uneasy%5:00:00:uncomfortable:01","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003926,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.309446042054333}
{"item_id":"senseval3.d000.s031.t000","gold_sense_keys":["get%2:30:02::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1918,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0003327,"input_uncached_usd":0.0000959,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9714807079872116}
{"item_id":"senseval3.d000.s031.t003","gold_sense_keys":["give%2:40:07::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval3.d000.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"give%2:40:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2034,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003897,"input_uncached_usd":0.00010169999999999999,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.685141999972984}
{"item_id":"senseval3.d000.s031.t005","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wait%2:31:00::","call_ids":["senseval3.d000.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wait%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020864999999999998,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.164402665977832}
{"item_id":"senseval3.d000.s031.t006","gold_sense_keys":["answer%1:10:01::"],"candidates":[{"index":1,"sense_key":"answer%1:10:01::","synset_id":"answer.n.01"},{"index":2,"sense_key":"answer%1:10:02::","synset_id":"solution.n.02"},{"index":3,"sense_key":"answer%1:10:00::","synset_id":"answer.n.03"},{"index":4,"sense_key":"answer%1:10:03::","synset_id":"answer.n.04"},{"index":5,"sense_key":"answer%1:04:00::","synset_id":"answer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%1:10:01::","call_ids":["senseval3.d000.s031.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041855,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.776729667035397}
{"item_id":"senseval3.d000.s031.t007","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028585,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.012049834011123}
{"item_id":"senseval3.d000.s031.t008","gold_sense_keys":["wink%2:29:00::"],"candidates":[{"index":1,"sense_key":"wink%2:29:02::","synset_id":"wink.v.01"},{"index":2,"sense_key":"wink%2:39:00::","synset_id":"flash.v.01"},{"index":3,"sense_key":"wink%2:29:00::","synset_id":"blink.v.01"},{"index":4,"sense_key":"wink%2:29:01::","synset_id":"wink.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wink%2:29:02::","call_ids":["senseval3.d000.s031.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wink%2:29:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054025,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.359154666017275}
{"item_id":"senseval3.d000.s031.t009","gold_sense_keys":["continue%2:42:01::"],"candidates":[{"index":1,"sense_key":"continue%2:42:01::","synset_id":"continue.v.01"},{"index":2,"sense_key":"continue%2:32:00::","synset_id":"continue.v.02"},{"index":3,"sense_key":"continue%2:42:00::","synset_id":"continue.v.03"},{"index":4,"sense_key":"continue%2:38:00::","synset_id":"proceed.v.02"},{"index":5,"sense_key":"continue%2:41:00::","synset_id":"retain.v.02"},{"index":6,"sense_key":"continue%2:42:12::","synset_id":"continue.v.06"},{"index":7,"sense_key":"continue%2:42:11::","synset_id":"continue.v.07"},{"index":8,"sense_key":"continue%2:42:10::","synset_id":"stay.v.04"},{"index":9,"sense_key":"continue%2:42:06::","synset_id":"cover.v.03"},{"index":10,"sense_key":"continue%2:30:10::","synset_id":"continue.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"continue%2:42:01::","call_ids":["senseval3.d000.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"continue%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003524,"input_uncached_usd":0.0000388,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1438994579948485}
{"item_id":"senseval3.d000.s031.t010","gold_sense_keys":["stare%2:39:00::"],"candidates":[{"index":1,"sense_key":"stare%2:39:00::","synset_id":"gaze.v.01"},{"index":2,"sense_key":"stare%2:29:00::","synset_id":"stare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stare%2:39:00::","call_ids":["senseval3.d000.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stare%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006388,"input_uncached_usd":0.000018,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.006434125010855}
{"item_id":"senseval3.d000.s032.t001","gold_sense_keys":["weak%5:00:00:perceptible:00"],"candidates":[{"index":1,"sense_key":"weak%3:00:00::","synset_id":"weak.a.01"},{"index":2,"sense_key":"weak%5:00:00:diluted:00","synset_id":"watery.s.04"},{"index":3,"sense_key":"weak%5:00:00:unstressed:00","synset_id":"unaccented.s.01"},{"index":4,"sense_key":"weak%5:00:00:human:00","synset_id":"fallible.s.01"},{"index":5,"sense_key":"weak%5:00:00:down:00","synset_id":"weak.s.04"},{"index":6,"sense_key":"weak%5:00:00:unskilled:00","synset_id":"weak.s.05"},{"index":7,"sense_key":"weak%5:00:00:frail:00","synset_id":"decrepit.s.02"},{"index":8,"sense_key":"weak%5:00:00:regular:00","synset_id":"weak.s.07"},{"index":9,"sense_key":"weak%5:00:00:powerless:00","synset_id":"weak.s.08"},{"index":10,"sense_key":"weak%5:00:00:perceptible:00","synset_id":"faint.s.01"},{"index":11,"sense_key":"weak%5:00:00:fallible:00","synset_id":"weak.s.10"},{"index":12,"sense_key":"weak%5:00:00:stupid:00","synset_id":"weak.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"weak%5:00:00:perceptible:00","call_ids":["senseval3.d000.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"weak%5:00:00:perceptible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020009999999999998,"input_uncached_usd":0.0000401,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9531785419676453}
{"item_id":"senseval3.d000.s032.t002","gold_sense_keys":["laugh%1:10:00::"],"candidates":[{"index":1,"sense_key":"laugh%1:10:00::","synset_id":"laugh.n.01"},{"index":2,"sense_key":"laugh%1:10:02::","synset_id":"laugh.n.02"},{"index":3,"sense_key":"laugh%1:10:01::","synset_id":"joke.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laugh%1:10:00::","call_ids":["senseval3.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laugh%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033295,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.681079250003677}
{"item_id":"senseval3.d000.s034.t000","gold_sense_keys":["thanks%1:10:00::"],"candidates":[{"index":1,"sense_key":"thanks%1:10:00::","synset_id":"thanks.n.01"},{"index":2,"sense_key":"thanks%1:04:00::","synset_id":"thanks.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thanks%1:10:00::","call_ids":["senseval3.d000.s034.t000__v1__a1","senseval3.d000.s034.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thanks%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037915,"input_uncached_usd":0.00003355,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.349006166972686}
{"item_id":"senseval3.d000.s035.t000","gold_sense_keys":["faint%5:00:00:perceptible:00"],"candidates":[{"index":1,"sense_key":"faint%5:00:00:perceptible:00","synset_id":"faint.s.01"},{"index":2,"sense_key":"faint%5:00:00:indistinct:00","synset_id":"dim.s.02"},{"index":3,"sense_key":"faint%5:00:00:weak:00","synset_id":"faint.s.03"},{"index":4,"sense_key":"faint%5:00:00:ill:01","synset_id":"faint.s.04"},{"index":5,"sense_key":"faint%5:00:02:indistinct:00","synset_id":"faint.s.05"},{"index":6,"sense_key":"faint%5:00:00:cowardly:00","synset_id":"faint.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"faint%5:00:00:perceptible:00","call_ids":["senseval3.d000.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"faint%5:00:00:perceptible:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003669,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.896818500012159}
{"item_id":"senseval3.d000.s035.t001","gold_sense_keys":["crease%1:25:01::"],"candidates":[{"index":1,"sense_key":"crease%1:25:02::","synset_id":"fold.n.01"},{"index":2,"sense_key":"crease%1:25:01::","synset_id":"wrinkle.n.01"},{"index":3,"sense_key":"crease%1:06:00::","synset_id":"kris.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crease%1:25:01::","call_ids":["senseval3.d000.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crease%1:25:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030765,"input_uncached_usd":0.00001965,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.8549363749916665}
{"item_id":"senseval3.d000.s035.t002","gold_sense_keys":["appear%2:30:00::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"appear%2:30:02::","call_ids":["senseval3.d000.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"appear%2:30:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002922,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.365762416971847}
{"item_id":"senseval3.d000.s035.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019939999999999997,"input_uncached_usd":0.000039399999999999995,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1642150830011815}
{"item_id":"senseval3.d000.s036.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022595,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3650207919999957}
{"item_id":"senseval3.d000.s036.t001","gold_sense_keys":["take%2:31:07::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"take%2:31:07::","call_ids":["senseval3.d000.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"take%2:31:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2102,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003931,"input_uncached_usd":0.0001051,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.155177291017026}
{"item_id":"senseval3.d000.s037.t000","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"mean%2:31:00::","call_ids":["senseval3.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"mean%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004929,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.321550916007254}
{"item_id":"senseval3.d000.s038.t000","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002634,"input_uncached_usd":0.000052199999999999995,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3919305829913355}
{"item_id":"senseval3.d000.s039.t000","gold_sense_keys":["hit%2:35:01::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d000.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hit%2:35:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003095,"input_uncached_usd":0.0000471,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.969959790993016}
{"item_id":"senseval3.d000.s039.t001","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":285,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0007886500000000001,"input_uncached_usd":0.000014249999999999999,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.373579458973836}
{"item_id":"senseval3.d000.s040.t000","gold_sense_keys":["return%2:40:03::"],"candidates":[{"index":1,"sense_key":"return%2:38:00::","synset_id":"return.v.01"},{"index":2,"sense_key":"return%2:40:08::","synset_id":"render.v.07"},{"index":3,"sense_key":"return%2:30:00::","synset_id":"revert.v.01"},{"index":4,"sense_key":"return%2:32:00::","synset_id":"hark_back.v.01"},{"index":5,"sense_key":"return%2:38:12::","synset_id":"return.v.05"},{"index":6,"sense_key":"return%2:40:03::","synset_id":"return.v.06"},{"index":7,"sense_key":"return%2:35:01::","synset_id":"return.v.07"},{"index":8,"sense_key":"return%2:32:01::","synset_id":"retort.v.01"},{"index":9,"sense_key":"return%2:30:01::","synset_id":"come_back.v.01"},{"index":10,"sense_key":"return%2:40:00::","synset_id":"refund.v.01"},{"index":11,"sense_key":"return%2:32:03::","synset_id":"render.v.05"},{"index":12,"sense_key":"return%2:41:00::","synset_id":"reelect.v.01"},{"index":13,"sense_key":"return%2:40:13::","synset_id":"fall.v.21"},{"index":14,"sense_key":"return%2:38:01::","synset_id":"return.v.14"},{"index":15,"sense_key":"return%2:36:00::","synset_id":"render.v.04"},{"index":16,"sense_key":"return%2:32:02::","synset_id":"return.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"return%2:40:03::","call_ids":["senseval3.d000.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"return%2:40:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003539,"input_uncached_usd":0.0000403,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.843084541964345}
{"item_id":"senseval3.d000.s040.t001","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025595,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.43374687503092}
{"item_id":"senseval3.d000.s041.t000","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032865,"input_uncached_usd":0.000040249999999999996,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.89548908296274}
{"item_id":"senseval3.d000.s041.t002","gold_sense_keys":["like%2:37:04::"],"candidates":[{"index":1,"sense_key":"like%2:37:01::","synset_id":"wish.v.02"},{"index":2,"sense_key":"like%2:37:05::","synset_id":"like.v.02"},{"index":3,"sense_key":"like%2:37:04::","synset_id":"like.v.03"},{"index":4,"sense_key":"like%2:31:00::","synset_id":"like.v.04"},{"index":5,"sense_key":"like%2:37:06::","synset_id":"like.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"like%2:37:04::","call_ids":["senseval3.d000.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"like%2:37:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001793,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1673972500138916}
{"item_id":"senseval3.d000.s043.t001","gold_sense_keys":["try%2:37:00::","try%2:41:01::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"try%2:41:01::","call_ids":["senseval3.d000.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"try%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032055,"input_uncached_usd":0.000032149999999999995,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.919625999988057}
{"item_id":"senseval3.d000.s044.t000","gold_sense_keys":["rub%2:35:00::"],"candidates":[{"index":1,"sense_key":"rub%2:35:00::","synset_id":"rub.v.01"},{"index":2,"sense_key":"rub%2:35:01::","synset_id":"rub.v.02"},{"index":3,"sense_key":"rub%2:39:00::","synset_id":"rub.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rub%2:35:00::","call_ids":["senseval3.d000.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rub%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":318,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001503,"input_uncached_usd":0.0000159,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.426660333003383}
{"item_id":"senseval3.d000.s044.t001","gold_sense_keys":["forehead%1:08:00::"],"candidates":[{"index":1,"sense_key":"forehead%1:08:00::","synset_id":"brow.n.01"},{"index":2,"sense_key":"forehead%1:08:01::","synset_id":"frontal_bone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"forehead%1:08:00::","call_ids":["senseval3.d000.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"forehead%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":288,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040480000000000003,"input_uncached_usd":0.0000144,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.117178832995705}
{"item_id":"senseval3.d000.s045.t001","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"feel%2:29:00::","call_ids":["senseval3.d000.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"feel%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045365,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.279527249978855}
{"item_id":"senseval3.d000.s045.t002","gold_sense_keys":["woolly%5:00:00:confused:00"],"candidates":[{"index":1,"sense_key":"woolly%5:00:00:soft:01","synset_id":"flocculent.s.01"},{"index":2,"sense_key":"woolly%5:00:00:confused:00","synset_id":"addled.s.02"},{"index":3,"sense_key":"woolly%5:00:00:hairy:00","synset_id":"wooly.s.03"},{"index":4,"sense_key":"woolly%5:00:02:hairy:00","synset_id":"lanate.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"woolly%5:00:00:confused:00","call_ids":["senseval3.d000.s045.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"woolly%5:00:00:confused:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005903500000000001,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.612402165948879}
{"item_id":"senseval3.d000.s046.t001","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":352,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017759999999999998,"input_uncached_usd":0.0000176,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5056845829822123}
{"item_id":"senseval3.d000.s046.t002","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":36,"chosen_sense_key":"make%2:38:10::","call_ids":["senseval3.d000.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":36,"predicted_sense_key":"make%2:38:10::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2176,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0008067999999999999,"input_uncached_usd":0.00010879999999999999,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.402709582995158}
{"item_id":"senseval3.d000.s046.t003","gold_sense_keys":["exit%1:04:00::"],"candidates":[{"index":1,"sense_key":"exit%1:06:00::","synset_id":"exit.n.01"},{"index":2,"sense_key":"exit%1:11:00::","synset_id":"passing.n.02"},{"index":3,"sense_key":"exit%1:04:00::","synset_id":"exit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"exit%1:04:00::","call_ids":["senseval3.d000.s046.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"exit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00009989999999999999,"input_uncached_usd":0.0000167,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.8300645829876885}
{"item_id":"senseval3.d000.s047.t000","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval3.d000.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00082465,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.996230792021379}
{"item_id":"senseval3.d000.s047.t001","gold_sense_keys":["come%2:30:13::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"come%2:30:13::","call_ids":["senseval3.d000.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"come%2:30:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00026645,"input_uncached_usd":0.000055249999999999994,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.197451665997505}
{"item_id":"senseval3.d000.s047.t003","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d000.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001805,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3676512080128305}
{"item_id":"senseval3.d000.s048.t000","gold_sense_keys":["concede%2:32:01::"],"candidates":[{"index":1,"sense_key":"concede%2:32:00::","synset_id":"concede.v.01"},{"index":2,"sense_key":"concede%2:32:01::","synset_id":"concede.v.02"},{"index":3,"sense_key":"concede%2:40:00::","synset_id":"concede.v.03"},{"index":4,"sense_key":"concede%2:33:00::","synset_id":"concede.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concede%2:32:01::","call_ids":["senseval3.d000.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concede%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.000385,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2235126249725}
{"item_id":"senseval3.d000.s048.t001","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval3.d000.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00053065,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.021522250026464}
{"item_id":"senseval3.d000.s049.t000","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d000.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1180,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002958,"input_uncached_usd":0.000059,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.899429249984678}
{"item_id":"senseval3.d000.s049.t001","gold_sense_keys":["ride%2:32:00::"],"candidates":[{"index":1,"sense_key":"ride%2:38:01::","synset_id":"ride.v.01"},{"index":2,"sense_key":"ride%2:38:00::","synset_id":"ride.v.02"},{"index":3,"sense_key":"ride%2:42:00::","synset_id":"ride.v.03"},{"index":4,"sense_key":"ride%2:38:02::","synset_id":"ride.v.04"},{"index":5,"sense_key":"ride%2:32:00::","synset_id":"tease.v.02"},{"index":6,"sense_key":"ride%2:42:04::","synset_id":"ride.v.06"},{"index":7,"sense_key":"ride%2:42:03::","synset_id":"drive.v.12"},{"index":8,"sense_key":"ride%2:42:07::","synset_id":"depend_on.v.01"},{"index":9,"sense_key":"ride%2:42:02::","synset_id":"ride.v.09"},{"index":10,"sense_key":"ride%2:38:04::","synset_id":"ride.v.10"},{"index":11,"sense_key":"ride%2:38:03::","synset_id":"ride.v.11"},{"index":12,"sense_key":"ride%2:38:07::","synset_id":"ride.v.12"},{"index":13,"sense_key":"ride%2:35:00::","synset_id":"ride.v.13"},{"index":14,"sense_key":"ride%2:35:03::","synset_id":"ride.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ride%2:32:00::","call_ids":["senseval3.d000.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ride%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020109999999999998,"input_uncached_usd":0.000041099999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.012061624962371}
{"item_id":"senseval3.d000.s049.t002","gold_sense_keys":["like%2:37:05::"],"candidates":[{"index":1,"sense_key":"like%2:37:01::","synset_id":"wish.v.02"},{"index":2,"sense_key":"like%2:37:05::","synset_id":"like.v.02"},{"index":3,"sense_key":"like%2:37:04::","synset_id":"like.v.03"},{"index":4,"sense_key":"like%2:31:00::","synset_id":"like.v.04"},{"index":5,"sense_key":"like%2:37:06::","synset_id":"like.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"like%2:37:05::","call_ids":["senseval3.d000.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"like%2:37:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015575,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.866842374962289}
{"item_id":"senseval3.d000.s049.t003","gold_sense_keys":["listen%2:39:00::"],"candidates":[{"index":1,"sense_key":"listen%2:39:00::","synset_id":"listen.v.01"},{"index":2,"sense_key":"listen%2:39:01::","synset_id":"listen.v.02"},{"index":3,"sense_key":"listen%2:41:11::","synset_id":"heed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"listen%2:39:00::","call_ids":["senseval3.d000.s049.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"listen%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005623,"input_uncached_usd":0.000018299999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.744284334010445}
{"item_id":"senseval3.d000.s049.t004","gold_sense_keys":["radio%1:06:01::"],"candidates":[{"index":1,"sense_key":"radio%1:10:00::","synset_id":"radio.n.01"},{"index":2,"sense_key":"radio%1:06:01::","synset_id":"radio_receiver.n.01"},{"index":3,"sense_key":"radio%1:06:00::","synset_id":"radio.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"radio%1:06:01::","call_ids":["senseval3.d000.s049.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"radio%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":352,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028,"input_uncached_usd":0.0000176,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.767295124998782}
{"item_id":"senseval3.d000.s049.t005","gold_sense_keys":["sing%2:36:00::"],"candidates":[{"index":1,"sense_key":"sing%2:36:01::","synset_id":"sing.v.01"},{"index":2,"sense_key":"sing%2:36:00::","synset_id":"sing.v.02"},{"index":3,"sense_key":"sing%2:32:08::","synset_id":"sing.v.03"},{"index":4,"sense_key":"sing%2:32:00::","synset_id":"whistle.v.05"},{"index":5,"sense_key":"sing%2:32:01::","synset_id":"spill_the_beans.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sing%2:36:00::","call_ids":["senseval3.d000.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sing%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025915,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.414085125026759}
{"item_id":"senseval3.d000.s049.t007","gold_sense_keys":["bath%1:04:00::"],"candidates":[{"index":1,"sense_key":"bath%1:06:02::","synset_id":"bath.n.01"},{"index":2,"sense_key":"bath%1:04:00::","synset_id":"bath.n.02"},{"index":3,"sense_key":"bath%1:06:00::","synset_id":"bathtub.n.01"},{"index":4,"sense_key":"bath%1:23:00::","synset_id":"bath.n.04"},{"index":5,"sense_key":"bath%1:15:00::","synset_id":"bath.n.05"},{"index":6,"sense_key":"bath%1:06:01::","synset_id":"bathroom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bath%1:04:00::","call_ids":["senseval3.d000.s049.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bath%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020984999999999998,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1238283749553375}
{"item_id":"senseval3.d000.s050.t000","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"say%2:32:01::","call_ids":["senseval3.d000.s050.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"say%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":736,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003759999999999999,"input_uncached_usd":0.0000368,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.479958250012714}
{"item_id":"senseval3.d000.s050.t001","gold_sense_keys":["neighbor%1:18:00::"],"candidates":[{"index":1,"sense_key":"neighbor%1:18:00::","synset_id":"neighbor.n.01"},{"index":2,"sense_key":"neighbor%1:17:00::","synset_id":"neighbor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"neighbor%1:18:00::","call_ids":["senseval3.d000.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"neighbor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007654,"input_uncached_usd":0.0000166,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.70923599996604}
{"item_id":"senseval3.d000.s050.t002","gold_sense_keys":["complain%2:32:00::"],"candidates":[{"index":1,"sense_key":"complain%2:32:00::","synset_id":"complain.v.01"},{"index":2,"sense_key":"complain%2:32:01::","synset_id":"complain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complain%2:32:00::","call_ids":["senseval3.d000.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033144999999999995,"input_uncached_usd":0.00001785,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.58273470797576}
{"item_id":"senseval3.d000.s050.t003","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval3.d000.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031165,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.014751875016373}
{"item_id":"senseval3.d000.s051.t000","gold_sense_keys":["tell%2:32:00::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval3.d000.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001381,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.478455832984764}
{"item_id":"senseval3.d000.s051.t001","gold_sense_keys":["bother%2:37:00::"],"candidates":[{"index":1,"sense_key":"bother%2:41:00::","synset_id":"trouble_oneself.v.01"},{"index":2,"sense_key":"bother%2:37:00::","synset_id":"annoy.v.01"},{"index":3,"sense_key":"bother%2:41:01::","synset_id":"trouble.v.02"},{"index":4,"sense_key":"bother%2:38:00::","synset_id":"bother.v.04"},{"index":5,"sense_key":"bother%2:37:03::","synset_id":"bother.v.05"},{"index":6,"sense_key":"bother%2:37:02::","synset_id":"bother.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bother%2:41:01::","call_ids":["senseval3.d000.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bother%2:41:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044269999999999997,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.757760125037748}
{"item_id":"senseval3.d000.s052.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002253,"input_uncached_usd":0.0000393,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.383672584022861}
{"item_id":"senseval3.d000.s052.t001","gold_sense_keys":["close%2:35:00::"],"candidates":[{"index":1,"sense_key":"close%2:35:00::","synset_id":"close.v.01"},{"index":2,"sense_key":"close%2:35:06::","synset_id":"close.v.02"},{"index":3,"sense_key":"close%2:41:00::","synset_id":"close_up.v.01"},{"index":4,"sense_key":"close%2:41:01::","synset_id":"close.v.04"},{"index":5,"sense_key":"close%2:42:00::","synset_id":"conclude.v.04"},{"index":6,"sense_key":"close%2:32:05::","synset_id":"close.v.06"},{"index":7,"sense_key":"close%2:42:01::","synset_id":"close.v.07"},{"index":8,"sense_key":"close%2:41:15::","synset_id":"close.v.08"},{"index":9,"sense_key":"close%2:39:10::","synset_id":"close.v.09"},{"index":10,"sense_key":"close%2:38:08::","synset_id":"close.v.10"},{"index":11,"sense_key":"close%2:38:10::","synset_id":"close.v.11"},{"index":12,"sense_key":"close%2:38:15::","synset_id":"close.v.12"},{"index":13,"sense_key":"close%2:35:14::","synset_id":"close.v.13"},{"index":14,"sense_key":"close%2:35:07::","synset_id":"close.v.14"},{"index":15,"sense_key":"close%2:35:11::","synset_id":"close.v.15"},{"index":16,"sense_key":"close%2:35:15::","synset_id":"close_up.v.03"},{"index":17,"sense_key":"close%2:30:08::","synset_id":"close.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"close%2:35:00::","call_ids":["senseval3.d000.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"close%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041069999999999995,"input_uncached_usd":0.0000459,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.731732291053049}
{"item_id":"senseval3.d000.s052.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002847,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.349697624973487}
{"item_id":"senseval3.d000.s052.t003","gold_sense_keys":["nod%2:32:00::"],"candidates":[{"index":1,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":2,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"},{"index":3,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":4,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":5,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"nod%2:29:03::","call_ids":["senseval3.d000.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"nod%2:29:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007193000000000001,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.547088540974073}
{"item_id":"senseval3.d000.s053.t000","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002964,"input_uncached_usd":0.000034,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.469853749964386}
{"item_id":"senseval3.d000.s053.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d000.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018099999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1961634169565514}
{"item_id":"senseval3.d000.s054.t000","gold_sense_keys":["fine%4:02:00::"],"candidates":[{"index":1,"sense_key":"fine%4:02:00::","synset_id":"very_well.r.02"},{"index":2,"sense_key":"fine%4:02:02::","synset_id":"finely.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%4:02:00::","call_ids":["senseval3.d000.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":302,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00027749999999999997,"input_uncached_usd":0.0000151,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.725826541020069}
{"item_id":"senseval3.d000.s055.t000","gold_sense_keys":["give%2:40:07::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"give%2:40:07::","call_ids":["senseval3.d000.s055.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"give%2:40:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1955,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00071855,"input_uncached_usd":0.00009774999999999999,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.126455582969356}
{"item_id":"senseval3.d000.s055.t001","gold_sense_keys":["address%1:15:00::"],"candidates":[{"index":1,"sense_key":"address%1:10:04::","synset_id":"address.n.01"},{"index":2,"sense_key":"address%1:15:00::","synset_id":"address.n.02"},{"index":3,"sense_key":"address%1:10:00::","synset_id":"address.n.03"},{"index":4,"sense_key":"address%1:10:03::","synset_id":"address.n.04"},{"index":5,"sense_key":"address%1:10:01::","synset_id":"address.n.05"},{"index":6,"sense_key":"address%1:10:02::","synset_id":"address.n.06"},{"index":7,"sense_key":"address%1:07:01::","synset_id":"address.n.07"},{"index":8,"sense_key":"address%1:07:00::","synset_id":"savoir-faire.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"address%1:15:00::","call_ids":["senseval3.d000.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"address%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002868,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.600845541979652}
{"item_id":"senseval3.d000.s056.t000","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d000.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2069,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00026345,"input_uncached_usd":0.00010345,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.975898207980208}
{"item_id":"senseval3.d000.s056.t001","gold_sense_keys":["time%1:03:00::","time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval3.d000.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005745,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.24559775000671}
{"item_id":"senseval3.d000.s057.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval3.d000.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002058,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.375036791956518}
{"item_id":"senseval3.d000.s057.t001","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["senseval3.d000.s057.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043985,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.6833470420097}
{"item_id":"senseval3.d000.s057.t003","gold_sense_keys":["habit%1:09:00::"],"candidates":[{"index":1,"sense_key":"habit%1:09:00::","synset_id":"habit.n.01"},{"index":2,"sense_key":"habit%1:04:02::","synset_id":"habit.n.02"},{"index":3,"sense_key":"habit%1:06:00::","synset_id":"habit.n.03"},{"index":4,"sense_key":"habit%1:22:00::","synset_id":"habit.n.04"},{"index":5,"sense_key":"habit%1:06:01::","synset_id":"habit.n.05"},{"index":6,"sense_key":"habit%1:04:01::","synset_id":"substance_abuse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"habit%1:04:02::","call_ids":["senseval3.d000.s057.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"habit%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025909999999999996,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.204060916963499}
{"item_id":"senseval3.d000.s057.t004","gold_sense_keys":["movement%1:04:04::"],"candidates":[{"index":1,"sense_key":"movement%1:04:00::","synset_id":"motion.n.03"},{"index":2,"sense_key":"movement%1:04:04::","synset_id":"motion.n.06"},{"index":3,"sense_key":"movement%1:11:00::","synset_id":"movement.n.03"},{"index":4,"sense_key":"movement%1:14:00::","synset_id":"movement.n.04"},{"index":5,"sense_key":"movement%1:10:00::","synset_id":"movement.n.05"},{"index":6,"sense_key":"movement%1:04:03::","synset_id":"campaign.n.02"},{"index":7,"sense_key":"movement%1:19:00::","synset_id":"apparent_motion.n.01"},{"index":8,"sense_key":"movement%1:22:00::","synset_id":"bowel_movement.n.01"},{"index":9,"sense_key":"movement%1:09:00::","synset_id":"drift.n.05"},{"index":10,"sense_key":"movement%1:06:00::","synset_id":"movement.n.10"},{"index":11,"sense_key":"movement%1:04:02::","synset_id":"movement.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"movement%1:04:00::","call_ids":["senseval3.d000.s057.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"movement%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00068395,"input_uncached_usd":0.00003755,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.801937374984846}
{"item_id":"senseval3.d000.s058.t000","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1350,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003555,"input_uncached_usd":0.0000675,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.15890862501692}
{"item_id":"senseval3.d000.s058.t001","gold_sense_keys":["make%2:30:00::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval3.d000.s058.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2164,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003706,"input_uncached_usd":0.0001082,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.488210415991489}
{"item_id":"senseval3.d000.s059.t000","gold_sense_keys":["suicide%1:04:00::"],"candidates":[{"index":1,"sense_key":"suicide%1:04:00::","synset_id":"suicide.n.01"},{"index":2,"sense_key":"suicide%1:18:00::","synset_id":"suicide.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suicide%1:04:00::","call_ids":["senseval3.d000.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suicide%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":278,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004043,"input_uncached_usd":0.000013899999999999999,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.48786445899168}
{"item_id":"senseval3.d000.s059.t001","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d000.s059.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011559999999999999,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.402529124985449}
{"item_id":"senseval3.d000.s060.t000","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d000.s060.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":2833,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.00115555,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0011332,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.937297583033796}
{"item_id":"senseval3.d000.s061.t001","gold_sense_keys":["fly%2:38:02::"],"candidates":[{"index":1,"sense_key":"fly%2:38:00::","synset_id":"fly.v.01"},{"index":2,"sense_key":"fly%2:38:02::","synset_id":"fly.v.02"},{"index":3,"sense_key":"fly%2:38:01::","synset_id":"fly.v.03"},{"index":4,"sense_key":"fly%2:35:00::","synset_id":"fly.v.04"},{"index":5,"sense_key":"fly%2:38:07::","synset_id":"fly.v.05"},{"index":6,"sense_key":"fly%2:38:08::","synset_id":"fly.v.06"},{"index":7,"sense_key":"fly%2:30:01::","synset_id":"fly.v.07"},{"index":8,"sense_key":"fly%2:38:06::","synset_id":"fly.v.08"},{"index":9,"sense_key":"fly%2:38:05::","synset_id":"fly.v.09"},{"index":10,"sense_key":"fly%2:39:00::","synset_id":"fly.v.10"},{"index":11,"sense_key":"fly%2:38:03::","synset_id":"flee.v.01"},{"index":12,"sense_key":"fly%2:38:04::","synset_id":"fly.v.12"},{"index":13,"sense_key":"fly%2:35:01::","synset_id":"fly.v.13"},{"index":14,"sense_key":"fly%2:30:00::","synset_id":"vanish.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fly%2:38:02::","call_ids":["senseval3.d000.s061.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fly%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024605,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5761702920426615}
{"item_id":"senseval3.d000.s063.t000","gold_sense_keys":["smile%2:29:00::"],"candidates":[{"index":1,"sense_key":"smile%2:29:00::","synset_id":"smile.v.01"},{"index":2,"sense_key":"smile%2:32:00::","synset_id":"smile.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"smile%2:29:00::","call_ids":["senseval3.d000.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"smile%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":264,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022439999999999998,"input_uncached_usd":0.000013199999999999999,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.01034187496407}
{"item_id":"senseval3.d000.s064.t001","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d000.s064.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032265,"input_uncached_usd":0.000034649999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.786382791993674}
{"item_id":"senseval3.d000.s065.t000","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d000.s065.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023505,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7957268750178628}
{"item_id":"senseval3.d000.s065.t001","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":28,"chosen_sense_key":"make%2:38:05::","call_ids":["senseval3.d000.s065.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":28,"predicted_sense_key":"make%2:38:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2190,"cached_input_tokens":0,"output_tokens":2512,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.0011143,"input_uncached_usd":0.00010949999999999999,"input_cached_usd":0.0,"output_usd":0.0010048,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.493563666008413}
{"item_id":"senseval3.d000.s065.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s065.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":296,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035399999999999993,"input_uncached_usd":0.000014799999999999999,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.157724332995713}
{"item_id":"senseval3.d000.s065.t004","gold_sense_keys":["stand_up%2:38:00::"],"candidates":[{"index":1,"sense_key":"stand_up%2:38:00::","synset_id":"arise.v.03"},{"index":2,"sense_key":"stand_up%2:33:00::","synset_id":"stand_up.v.02"},{"index":3,"sense_key":"stand_up%2:35:03::","synset_id":"stand.v.10"},{"index":4,"sense_key":"stand_up%2:35:00::","synset_id":"stand.v.01"},{"index":5,"sense_key":"stand_up%2:32:00::","synset_id":"stand_up.v.05"},{"index":6,"sense_key":"stand_up%2:42:00::","synset_id":"stand_up.v.06"},{"index":7,"sense_key":"stand_up%2:38:06::","synset_id":"bristle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stand_up%2:35:03::","call_ids":["senseval3.d000.s065.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stand_up%2:35:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002894,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.000705124984961}
{"item_id":"senseval3.d000.s065.t005","gold_sense_keys":["nod%2:32:01::"],"candidates":[{"index":1,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":2,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"},{"index":3,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":4,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":5,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nod%2:32:01::","call_ids":["senseval3.d000.s065.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nod%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043694999999999997,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.064729499979876}
{"item_id":"senseval3.d000.s065.t006","gold_sense_keys":["farewell%1:10:00::"],"candidates":[{"index":1,"sense_key":"farewell%1:10:00::","synset_id":"farewell.n.01"},{"index":2,"sense_key":"farewell%1:04:00::","synset_id":"farewell.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"farewell%1:10:00::","call_ids":["senseval3.d000.s065.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"farewell%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":313,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038044999999999995,"input_uncached_usd":0.00001565,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.194886832963675}
{"item_id":"senseval3.d000.s066.t000","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d000.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.0009819499999999999,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.103127749986015}
{"item_id":"senseval3.d000.s066.t001","gold_sense_keys":["small%3:00:00::"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"small%5:00:00:limited:00","call_ids":["senseval3.d000.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"small%5:00:00:limited:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024595,"input_uncached_usd":0.00003475,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.036664583021775}
{"item_id":"senseval3.d000.s066.t002","gold_sense_keys":["wiry%5:00:00:thin:03"],"candidates":[{"index":1,"sense_key":"wiry%5:00:00:thin:03","synset_id":"stringy.s.01"},{"index":2,"sense_key":"wiry%3:01:00::","synset_id":"wiry.a.02"},{"index":3,"sense_key":"wiry%5:00:00:hairy:00","synset_id":"wiry.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wiry%5:00:00:thin:03","call_ids":["senseval3.d000.s066.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wiry%5:00:00:thin:03","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015155,"input_uncached_usd":0.00001715,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9721383749856614}
{"item_id":"senseval3.d000.s066.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s066.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":794,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025089999999999997,"input_uncached_usd":0.000039699999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.972275542037096}
{"item_id":"senseval3.d000.s066.t004","gold_sense_keys":["slip%2:38:01::"],"candidates":[{"index":1,"sense_key":"slip%2:38:01::","synset_id":"steal.v.02"},{"index":2,"sense_key":"slip%2:30:04::","synset_id":"slip.v.02"},{"index":3,"sense_key":"slip%2:38:00::","synset_id":"skid.v.04"},{"index":4,"sense_key":"slip%2:30:00::","synset_id":"slip.v.04"},{"index":5,"sense_key":"slip%2:38:05::","synset_id":"slip.v.05"},{"index":6,"sense_key":"slip%2:31:00::","synset_id":"err.v.01"},{"index":7,"sense_key":"slip%2:40:10::","synset_id":"slip.v.07"},{"index":8,"sense_key":"slip%2:38:02::","synset_id":"slip.v.08"},{"index":9,"sense_key":"slip%2:38:12::","synset_id":"slip.v.09"},{"index":10,"sense_key":"slip%2:31:01::","synset_id":"slip.v.10"},{"index":11,"sense_key":"slip%2:30:03::","synset_id":"dislocate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"slip%2:38:01::","call_ids":["senseval3.d000.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"slip%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00070605,"input_uncached_usd":0.00003405,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.87885570799699}
{"item_id":"senseval3.d000.s066.t005","gold_sense_keys":["door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"door%1:06:00::","call_ids":["senseval3.d000.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"door%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005691,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.869365833001211}
{"item_id":"senseval3.d000.s066.t006","gold_sense_keys":["feel%2:29:00::","feel%2:31:01::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007377,"input_uncached_usd":0.0000401,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.568148875026964}
{"item_id":"senseval3.d000.s066.t008","gold_sense_keys":["see%2:31:02::","see%2:32:00::","see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s066.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1401,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00053725,"input_uncached_usd":0.00007005,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.194244374986738}
{"item_id":"senseval3.d000.s066.t009","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d000.s066.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031005000000000003,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.613140833040234}
{"item_id":"senseval3.d000.s066.t010","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s066.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023409999999999997,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.216765624994878}
{"item_id":"senseval3.d000.s066.t011","gold_sense_keys":["departure%1:04:00::"],"candidates":[{"index":1,"sense_key":"departure%1:04:00::","synset_id":"departure.n.01"},{"index":2,"sense_key":"departure%1:11:01::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"departure%1:11:00::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"departure%1:04:00::","call_ids":["senseval3.d000.s066.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"departure%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022994999999999998,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.990212625009008}
{"item_id":"senseval3.d000.s067.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d000.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004894999999999999,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.224737417011056}
{"item_id":"senseval3.d000.s067.t001","gold_sense_keys":["stay%2:38:00::","stay%2:42:01::","stay%2:42:02::"],"candidates":[{"index":1,"sense_key":"stay%2:30:00::","synset_id":"stay.v.01"},{"index":2,"sense_key":"stay%2:38:00::","synset_id":"stay.v.02"},{"index":3,"sense_key":"stay%2:42:01::","synset_id":"bide.v.01"},{"index":4,"sense_key":"stay%2:42:02::","synset_id":"stay.v.04"},{"index":5,"sense_key":"stay%2:38:01::","synset_id":"stay.v.05"},{"index":6,"sense_key":"stay%2:30:01::","synset_id":"stay.v.06"},{"index":7,"sense_key":"stay%2:42:04::","synset_id":"persist.v.03"},{"index":8,"sense_key":"stay%2:42:03::","synset_id":"last_out.v.01"},{"index":9,"sense_key":"stay%2:41:00::","synset_id":"stay.v.09"},{"index":10,"sense_key":"stay%2:35:00::","synset_id":"stay.v.10"},{"index":11,"sense_key":"stay%2:34:00::","synset_id":"quell.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stay%2:38:00::","call_ids":["senseval3.d000.s067.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stay%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050345,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.780857499979902}
{"item_id":"senseval3.d000.s067.t002","gold_sense_keys":["time%1:03:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"time%1:03:00::","call_ids":["senseval3.d000.s067.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"time%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00078285,"input_uncached_usd":0.00003405,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.342155999969691}
{"item_id":"senseval3.d000.s067.t003","gold_sense_keys":["pass%2:38:03::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"pass%2:38:03::","call_ids":["senseval3.d000.s067.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"pass%2:38:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1415,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00035875,"input_uncached_usd":0.00007075,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.819415417034179}
{"item_id":"senseval3.d000.s067.t004","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d000.s067.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004902,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.939687499951106}
{"item_id":"senseval3.d000.s067.t005","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":2,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"},{"index":3,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"strange%3:00:01::","call_ids":["senseval3.d000.s067.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"strange%3:00:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004115,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.498723709024489}
{"item_id":"senseval3.d000.s067.t006","gold_sense_keys":["little%3:00:00::","little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"little%3:00:00::","call_ids":["senseval3.d000.s067.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"little%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00078185,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.899871499976143}
{"item_id":"senseval3.d000.s067.t007","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s067.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032865,"input_uncached_usd":0.00004065,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.899209416995291}
{"item_id":"senseval3.d000.s067.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s067.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004133,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.634891458030324}
{"item_id":"senseval3.d000.s067.t009","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s067.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004674999999999999,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.17664987500757}
{"item_id":"senseval3.d000.s067.t010","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s067.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001606,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.111009082989767}
{"item_id":"senseval3.d000.s068.t002","gold_sense_keys":["return%2:38:00::"],"candidates":[{"index":1,"sense_key":"return%2:38:00::","synset_id":"return.v.01"},{"index":2,"sense_key":"return%2:40:08::","synset_id":"render.v.07"},{"index":3,"sense_key":"return%2:30:00::","synset_id":"revert.v.01"},{"index":4,"sense_key":"return%2:32:00::","synset_id":"hark_back.v.01"},{"index":5,"sense_key":"return%2:38:12::","synset_id":"return.v.05"},{"index":6,"sense_key":"return%2:40:03::","synset_id":"return.v.06"},{"index":7,"sense_key":"return%2:35:01::","synset_id":"return.v.07"},{"index":8,"sense_key":"return%2:32:01::","synset_id":"retort.v.01"},{"index":9,"sense_key":"return%2:30:01::","synset_id":"come_back.v.01"},{"index":10,"sense_key":"return%2:40:00::","synset_id":"refund.v.01"},{"index":11,"sense_key":"return%2:32:03::","synset_id":"render.v.05"},{"index":12,"sense_key":"return%2:41:00::","synset_id":"reelect.v.01"},{"index":13,"sense_key":"return%2:40:13::","synset_id":"fall.v.21"},{"index":14,"sense_key":"return%2:38:01::","synset_id":"return.v.14"},{"index":15,"sense_key":"return%2:36:00::","synset_id":"render.v.04"},{"index":16,"sense_key":"return%2:32:02::","synset_id":"return.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"return%2:38:00::","call_ids":["senseval3.d000.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"return%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025564999999999995,"input_uncached_usd":0.000044449999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2586899580201134}
{"item_id":"senseval3.d000.s068.t005","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval3.d000.s068.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037259999999999995,"input_uncached_usd":0.0000334,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.447123708028812}
{"item_id":"senseval3.d000.s068.t006","gold_sense_keys":["whole%3:00:00::"],"candidates":[{"index":1,"sense_key":"whole%3:00:00::","synset_id":"whole.a.01"},{"index":2,"sense_key":"whole%3:00:02::","synset_id":"whole.a.02"},{"index":3,"sense_key":"whole%5:00:00:uninjured:00","synset_id":"unharmed.s.01"},{"index":4,"sense_key":"whole%5:00:00:healthy:00","synset_id":"hale.s.01"},{"index":5,"sense_key":"whole%5:00:00:undiversified:00","synset_id":"solid.s.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whole%3:00:00::","call_ids":["senseval3.d000.s068.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whole%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015949999999999997,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7389827080187388}
{"item_id":"senseval3.d000.s068.t007","gold_sense_keys":["incident%1:11:00::"],"candidates":[{"index":1,"sense_key":"incident%1:11:00::","synset_id":"incident.n.01"},{"index":2,"sense_key":"incident%1:26:00::","synset_id":"incident.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incident%1:11:00::","call_ids":["senseval3.d000.s068.t007__v1__a1","senseval3.d000.s068.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incident%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040794999999999997,"input_uncached_usd":0.00003675,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.991628041025251}
{"item_id":"senseval3.d000.s068.t008","gold_sense_keys":["mind%1:09:00::","mind%1:09:01::","mind%1:09:03::"],"candidates":[{"index":1,"sense_key":"mind%1:09:00::","synset_id":"mind.n.01"},{"index":2,"sense_key":"mind%1:09:01::","synset_id":"mind.n.02"},{"index":3,"sense_key":"mind%1:09:04::","synset_id":"judgment.n.01"},{"index":4,"sense_key":"mind%1:18:00::","synset_id":"thinker.n.01"},{"index":5,"sense_key":"mind%1:09:03::","synset_id":"mind.n.05"},{"index":6,"sense_key":"mind%1:09:06::","synset_id":"mind.n.06"},{"index":7,"sense_key":"mind%1:09:05::","synset_id":"mind.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mind%1:09:01::","call_ids":["senseval3.d000.s068.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mind%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036844999999999993,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.262296332977712}
{"item_id":"senseval3.d000.s069.t000","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d000.s069.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000797,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.5787685420364141}
{"item_id":"senseval3.d000.s069.t001","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"day%1:28:03::","call_ids":["senseval3.d000.s069.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"day%1:28:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002501,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.082839208014775}
{"item_id":"senseval3.d000.s069.t002","gold_sense_keys":["hangover%1:26:00::"],"candidates":[{"index":1,"sense_key":"hangover%1:26:00::","synset_id":"hangover.n.01"},{"index":2,"sense_key":"hangover%1:18:00::","synset_id":"holdover.n.01"},{"index":3,"sense_key":"hangover%1:04:00::","synset_id":"hangover.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hangover%1:26:00::","call_ids":["senseval3.d000.s069.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hangover%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003094,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.027983582986053}
{"item_id":"senseval3.d000.s069.t003","gold_sense_keys":["remind%2:31:00::"],"candidates":[{"index":1,"sense_key":"remind%2:31:00::","synset_id":"remind.v.01"},{"index":2,"sense_key":"remind%2:32:00::","synset_id":"prompt.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remind%2:31:00::","call_ids":["senseval3.d000.s069.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remind%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033289999999999996,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.731514125014655}
{"item_id":"senseval3.d000.s069.t004","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"night%1:28:00::","call_ids":["senseval3.d000.s069.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"night%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005975,"input_uncached_usd":0.000027899999999999997,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.42433983302908}
{"item_id":"senseval3.d000.s070.t000","gold_sense_keys":["hangover%1:26:00::"],"candidates":[{"index":1,"sense_key":"hangover%1:26:00::","synset_id":"hangover.n.01"},{"index":2,"sense_key":"hangover%1:18:00::","synset_id":"holdover.n.01"},{"index":3,"sense_key":"hangover%1:04:00::","synset_id":"hangover.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hangover%1:26:00::","call_ids":["senseval3.d000.s070.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hangover%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036224999999999994,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.968677458993625}
{"item_id":"senseval3.d000.s070.t001","gold_sense_keys":["remind%2:31:00::"],"candidates":[{"index":1,"sense_key":"remind%2:31:00::","synset_id":"remind.v.01"},{"index":2,"sense_key":"remind%2:32:00::","synset_id":"prompt.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remind%2:31:00::","call_ids":["senseval3.d000.s070.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remind%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030935,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.690461042046081}
{"item_id":"senseval3.d000.s071.t000","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["senseval3.d000.s071.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1724,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005538,"input_uncached_usd":0.0000862,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.078352833981626}
{"item_id":"senseval3.d000.s071.t002","gold_sense_keys":["day%1:28:04::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"day%1:28:02::","call_ids":["senseval3.d000.s071.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"day%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045545,"input_uncached_usd":0.00003945,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5593219170114025}
{"item_id":"senseval3.d000.s071.t003","gold_sense_keys":["pass%2:38:05::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pass%2:38:05::","call_ids":["senseval3.d000.s071.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pass%2:38:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1465,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00036125,"input_uncached_usd":0.00007324999999999999,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.207063542038668}
{"item_id":"senseval3.d000.s071.t007","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":2,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":3,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":4,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":5,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":7,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d000.s071.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004977,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.76622116699582}
{"item_id":"senseval3.d000.s071.t009","gold_sense_keys":["tenant%1:18:00::"],"candidates":[{"index":1,"sense_key":"tenant%1:18:00::","synset_id":"tenant.n.01"},{"index":2,"sense_key":"tenant%1:18:02::","synset_id":"tenant.n.02"},{"index":3,"sense_key":"tenant%1:18:01::","synset_id":"tenant.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"tenant%1:18:01::","call_ids":["senseval3.d000.s071.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"tenant%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025875,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.091538499982562}
{"item_id":"senseval3.d000.s071.t010","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["senseval3.d000.s071.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004911,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.477686750004068}
{"item_id":"senseval3.d000.s071.t011","gold_sense_keys":["life%1:26:02::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"life%1:26:00::","call_ids":["senseval3.d000.s071.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"life%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005374500000000001,"input_uncached_usd":0.00004465,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.728578750044107}
{"item_id":"senseval3.d000.s072.t001","gold_sense_keys":["ignore%2:32:02::"],"candidates":[{"index":1,"sense_key":"ignore%2:32:02::","synset_id":"ignore.v.01"},{"index":2,"sense_key":"ignore%2:32:00::","synset_id":"dismiss.v.01"},{"index":3,"sense_key":"ignore%2:39:00::","synset_id":"ignore.v.03"},{"index":4,"sense_key":"ignore%2:31:03::","synset_id":"neglect.v.04"},{"index":5,"sense_key":"ignore%2:31:01::","synset_id":"ignore.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ignore%2:32:02::","call_ids":["senseval3.d000.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ignore%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061885,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.707216040987987}
{"item_id":"senseval3.d000.s074.t001","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":2,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":3,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":4,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":5,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":7,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d000.s074.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00072575,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.005320416996256}
{"item_id":"senseval3.d000.s075.t001","gold_sense_keys":["mail%2:35:00::"],"candidates":[{"index":1,"sense_key":"mail%2:35:00::","synset_id":"mail.v.01"},{"index":2,"sense_key":"mail%2:32:00::","synset_id":"mail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mail%2:35:00::","call_ids":["senseval3.d000.s075.t001__v1__a1","senseval3.d000.s075.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mail%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006404500000000001,"input_uncached_usd":0.000038849999999999996,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.99421766703017}
{"item_id":"senseval3.d000.s075.t003","gold_sense_keys":["check%1:21:00::"],"candidates":[{"index":1,"sense_key":"check%1:21:00::","synset_id":"check.n.01"},{"index":2,"sense_key":"check%1:09:00::","synset_id":"assay.n.01"},{"index":3,"sense_key":"check%1:10:00::","synset_id":"check.n.03"},{"index":4,"sense_key":"check%1:26:00::","synset_id":"arrest.n.02"},{"index":5,"sense_key":"check%1:09:01::","synset_id":"confirmation.n.01"},{"index":6,"sense_key":"check%1:04:05::","synset_id":"check.n.06"},{"index":7,"sense_key":"check%1:10:01::","synset_id":"check_mark.n.01"},{"index":8,"sense_key":"check%1:09:02::","synset_id":"hindrance.n.01"},{"index":9,"sense_key":"check%1:07:00::","synset_id":"check.n.09"},{"index":10,"sense_key":"check%1:06:00::","synset_id":"check.n.10"},{"index":11,"sense_key":"check%1:04:01::","synset_id":"bridle.n.02"},{"index":12,"sense_key":"check%1:04:02::","synset_id":"check.n.12"},{"index":13,"sense_key":"check%1:04:04::","synset_id":"check.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"check%1:21:00::","call_ids":["senseval3.d000.s075.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"check%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027899999999999995,"input_uncached_usd":0.000042199999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.368472582951654}
{"item_id":"senseval3.d000.s075.t005","gold_sense_keys":["maintenance%1:21:01::"],"candidates":[{"index":1,"sense_key":"maintenance%1:04:00::","synset_id":"care.n.06"},{"index":2,"sense_key":"maintenance%1:21:00::","synset_id":"maintenance.n.02"},{"index":3,"sense_key":"maintenance%1:21:01::","synset_id":"alimony.n.01"},{"index":4,"sense_key":"maintenance%1:04:01::","synset_id":"sustenance.n.03"},{"index":5,"sense_key":"maintenance%1:04:02::","synset_id":"maintenance.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"maintenance%1:21:01::","call_ids":["senseval3.d000.s075.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"maintenance%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013524999999999998,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.540752833010629}
{"item_id":"senseval3.d000.s075.t007","gold_sense_keys":["wish%2:32:02::","wish%2:37:00::"],"candidates":[{"index":1,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":2,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":3,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"},{"index":4,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":5,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":6,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"wish%2:37:01::","call_ids":["senseval3.d000.s075.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"wish%2:37:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005687499999999999,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.554533042013645}
{"item_id":"senseval3.d000.s075.t008","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s075.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":1298,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005363499999999999,"input_uncached_usd":0.00001715,"input_cached_usd":0.0,"output_usd":0.0005191999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.996955125010572}
{"item_id":"senseval3.d000.s075.t009","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s075.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":3664,"reasoning_output_tokens":3648},"cost":{"currency":"USD","total_usd":0.0015094499999999999,"input_uncached_usd":0.000043849999999999995,"input_cached_usd":0.0,"output_usd":0.0014655999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":23.532961792021524}
{"item_id":"senseval3.d000.s076.t000","gold_sense_keys":["come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d000.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1146,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00042209999999999996,"input_uncached_usd":0.0000573,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.992021834012121}
{"item_id":"senseval3.d000.s076.t001","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d000.s076.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005527,"input_uncached_usd":0.0000343,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.923897541011684}
{"item_id":"senseval3.d000.s076.t002","gold_sense_keys":["work%1:06:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%1:04:01::","call_ids":["senseval3.d000.s076.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019164999999999997,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5737995409872383}
{"item_id":"senseval3.d000.s076.t004","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s076.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1417,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00023085,"input_uncached_usd":0.00007085,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1446850830106996}
{"item_id":"senseval3.d000.s077.t000","gold_sense_keys":["lobby%1:06:00::"],"candidates":[{"index":1,"sense_key":"lobby%1:06:00::","synset_id":"anteroom.n.01"},{"index":2,"sense_key":"lobby%1:14:01::","synset_id":"lobby.n.02"},{"index":3,"sense_key":"lobby%1:14:00::","synset_id":"lobby.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lobby%1:06:00::","call_ids":["senseval3.d000.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lobby%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.000127,"input_uncached_usd":0.0000182,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1482235419680364}
{"item_id":"senseval3.d000.s077.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d000.s077.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005401000000000001,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.593657999997959}
{"item_id":"senseval3.d000.s077.t002","gold_sense_keys":["stand%2:35:00::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d000.s077.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002981,"input_uncached_usd":0.0000357,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.571133584016934}
{"item_id":"senseval3.d000.s077.t003","gold_sense_keys":["talk%2:32:01::"],"candidates":[{"index":1,"sense_key":"talk%2:32:01::","synset_id":"talk.v.01"},{"index":2,"sense_key":"talk%2:32:00::","synset_id":"talk.v.02"},{"index":3,"sense_key":"talk%2:32:03::","synset_id":"speak.v.03"},{"index":4,"sense_key":"talk%2:32:02::","synset_id":"spill.v.05"},{"index":5,"sense_key":"talk%2:32:06::","synset_id":"spill_the_beans.v.01"},{"index":6,"sense_key":"talk%2:32:05::","synset_id":"lecture.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talk%2:32:01::","call_ids":["senseval3.d000.s077.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talk%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":1618,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006734999999999999,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.0006472,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.576142249978147}
{"item_id":"senseval3.d000.s078.t000","gold_sense_keys":["spine%1:08:00::"],"candidates":[{"index":1,"sense_key":"spine%1:08:00::","synset_id":"spinal_column.n.01"},{"index":2,"sense_key":"spine%1:25:00::","synset_id":"spur.n.02"},{"index":3,"sense_key":"spine%1:20:00::","synset_id":"spine.n.03"},{"index":4,"sense_key":"spine%1:06:00::","synset_id":"spine.n.04"},{"index":5,"sense_key":"spine%1:05:00::","synset_id":"spine.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spine%1:08:00::","call_ids":["senseval3.d000.s078.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spine%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023639999999999997,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8776027080020867}
{"item_id":"senseval3.d000.s078.t001","gold_sense_keys":["crawl%2:42:01::"],"candidates":[{"index":1,"sense_key":"crawl%2:38:00::","synset_id":"crawl.v.01"},{"index":2,"sense_key":"crawl%2:42:01::","synset_id":"crawl.v.02"},{"index":3,"sense_key":"crawl%2:42:00::","synset_id":"crawl.v.03"},{"index":4,"sense_key":"crawl%2:38:03::","synset_id":"fawn.v.01"},{"index":5,"sense_key":"crawl%2:38:01::","synset_id":"crawl.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"crawl%2:42:01::","call_ids":["senseval3.d000.s078.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"crawl%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025919999999999996,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.551331542024855}
{"item_id":"senseval3.d000.s078.t003","gold_sense_keys":["premonition%1:12:00::"],"candidates":[{"index":1,"sense_key":"premonition%1:12:00::","synset_id":"foreboding.n.01"},{"index":2,"sense_key":"premonition%1:10:00::","synset_id":"forewarning.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"premonition%1:12:00::","call_ids":["senseval3.d000.s078.t003__v1__a1","senseval3.d000.s078.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"premonition%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":608,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00028054999999999996,"input_uncached_usd":0.00003735,"input_cached_usd":0.0,"output_usd":0.00024319999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.037349041958805}
{"item_id":"senseval3.d000.s078.t004","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval3.d000.s078.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031635,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.583152875013184}
{"item_id":"senseval3.d000.s078.t005","gold_sense_keys":["fellow%1:18:03::"],"candidates":[{"index":1,"sense_key":"fellow%1:18:00::","synset_id":"chap.n.01"},{"index":2,"sense_key":"fellow%1:18:02::","synset_id":"companion.n.01"},{"index":3,"sense_key":"fellow%1:18:03::","synset_id":"colleague.n.02"},{"index":4,"sense_key":"fellow%1:23:00::","synset_id":"mate.n.06"},{"index":5,"sense_key":"fellow%1:18:05::","synset_id":"fellow.n.05"},{"index":6,"sense_key":"fellow%1:18:04::","synset_id":"fellow.n.06"},{"index":7,"sense_key":"fellow%1:18:01::","synset_id":"boyfriend.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fellow%1:18:03::","call_ids":["senseval3.d000.s078.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fellow%1:18:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047129999999999996,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.097864541981835}
{"item_id":"senseval3.d000.s078.t006","gold_sense_keys":["tenant%1:18:00::"],"candidates":[{"index":1,"sense_key":"tenant%1:18:00::","synset_id":"tenant.n.01"},{"index":2,"sense_key":"tenant%1:18:02::","synset_id":"tenant.n.02"},{"index":3,"sense_key":"tenant%1:18:01::","synset_id":"tenant.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tenant%1:18:00::","call_ids":["senseval3.d000.s078.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tenant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":1236,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005128,"input_uncached_usd":0.0000184,"input_cached_usd":0.0,"output_usd":0.0004944,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.9963730000308715}
{"item_id":"senseval3.d000.s078.t007","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":3,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":4,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":5,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"happen%2:30:02::","call_ids":["senseval3.d000.s078.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"happen%2:30:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046704999999999994,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.200666124990676}
{"item_id":"senseval3.d000.s079.t001","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":2,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":3,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":4,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":5,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":6,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":7,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":8,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":9,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":10,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":11,"sense_key":"die%2:30:02::","synset_id":"die.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval3.d000.s079.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00012009999999999999,"input_uncached_usd":0.000036899999999999996,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0025078330072574}
{"item_id":"senseval3.d000.s080.t000","gold_sense_keys":["afternoon%1:28:00::"],"candidates":[{"index":1,"sense_key":"afternoon%1:28:00::","synset_id":"afternoon.n.01"},{"index":2,"sense_key":"afternoon%1:10:00::","synset_id":"good_afternoon.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"afternoon%1:28:00::","call_ids":["senseval3.d000.s080.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"afternoon%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030475,"input_uncached_usd":0.00001675,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.169416082964744}
{"item_id":"senseval3.d000.s080.t001","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d000.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036174999999999993,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.830608832999133}
{"item_id":"senseval3.d000.s080.t002","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":2,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":3,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":4,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":5,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":6,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":7,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":8,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":9,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":10,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":11,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":12,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":13,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":14,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":15,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":16,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":17,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":18,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":19,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":20,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":21,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":23,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":24,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":25,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":26,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":27,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":28,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":29,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":30,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":31,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":32,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"fall%2:38:15::","call_ids":["senseval3.d000.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"fall%2:38:15::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1528,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004412,"input_uncached_usd":0.0000764,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.63212783302879}
{"item_id":"senseval3.d000.s080.t003","gold_sense_keys":["roof%1:06:00::"],"candidates":[{"index":1,"sense_key":"roof%1:06:00::","synset_id":"roof.n.01"},{"index":2,"sense_key":"roof%1:06:01::","synset_id":"roof.n.02"},{"index":3,"sense_key":"roof%1:17:00::","synset_id":"roof.n.03"},{"index":4,"sense_key":"roof%1:10:00::","synset_id":"ceiling.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"roof%1:06:00::","call_ids":["senseval3.d000.s080.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"roof%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028369999999999996,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.853579375019763}
{"item_id":"senseval3.d000.s080.t004","gold_sense_keys":["obscure%5:00:00:unclear:00"],"candidates":[{"index":1,"sense_key":"obscure%5:00:00:unclear:00","synset_id":"obscure.s.01"},{"index":2,"sense_key":"obscure%5:00:00:incomprehensible:00","synset_id":"dark.s.06"},{"index":3,"sense_key":"obscure%5:00:00:concealed:00","synset_id":"hidden.s.03"},{"index":4,"sense_key":"obscure%5:00:00:inglorious:00","synset_id":"obscure.s.04"},{"index":5,"sense_key":"obscure%5:00:00:inconspicuous:00","synset_id":"obscure.s.05"},{"index":6,"sense_key":"obscure%5:00:00:unconnected:00","synset_id":"apart.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obscure%5:00:00:unclear:00","call_ids":["senseval3.d000.s080.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obscure%5:00:00:unclear:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005232,"input_uncached_usd":0.0000304,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.001129999989644}
{"item_id":"senseval3.d000.s082.t001","gold_sense_keys":["witness%2:39:00::"],"candidates":[{"index":1,"sense_key":"witness%2:39:00::","synset_id":"witness.v.01"},{"index":2,"sense_key":"witness%2:39:01::","synset_id":"witness.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"witness%2:39:01::","call_ids":["senseval3.d000.s082.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"witness%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045894999999999996,"input_uncached_usd":0.000017349999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.901788332965225}
{"item_id":"senseval3.d000.s082.t002","gold_sense_keys":["fall%1:11:00::"],"candidates":[{"index":1,"sense_key":"fall%1:28:00::","synset_id":"fall.n.01"},{"index":2,"sense_key":"fall%1:04:01::","synset_id":"spill.n.04"},{"index":3,"sense_key":"fall%1:11:03::","synset_id":"fall.n.03"},{"index":4,"sense_key":"fall%1:17:00::","synset_id":"descent.n.05"},{"index":5,"sense_key":"fall%1:04:03::","synset_id":"fall.n.05"},{"index":6,"sense_key":"fall%1:11:01::","synset_id":"fall.n.06"},{"index":7,"sense_key":"fall%1:11:04::","synset_id":"fall.n.07"},{"index":8,"sense_key":"fall%1:04:02::","synset_id":"capitulation.n.03"},{"index":9,"sense_key":"fall%1:28:01::","synset_id":"twilight.n.01"},{"index":10,"sense_key":"fall%1:11:02::","synset_id":"fall.n.10"},{"index":11,"sense_key":"fall%1:11:00::","synset_id":"drop.n.06"},{"index":12,"sense_key":"fall%1:07:00::","synset_id":"drop.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"fall%1:11:00::","call_ids":["senseval3.d000.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"fall%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005569500000000001,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.400011291028932}
{"item_id":"senseval3.d000.s082.t004","gold_sense_keys":["impact%1:11:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"impact%1:11:00::","call_ids":["senseval3.d000.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"impact%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023234999999999998,"input_uncached_usd":0.00002115,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4976223330013454}
{"item_id":"senseval3.d000.s082.t005","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"body%1:08:02::","call_ids":["senseval3.d000.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"body%1:08:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019559999999999998,"input_uncached_usd":0.0000356,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.516267541970592}
{"item_id":"senseval3.d000.s082.t006","gold_sense_keys":["smash%2:35:05::"],"candidates":[{"index":1,"sense_key":"smash%2:35:00::","synset_id":"smash.v.01"},{"index":2,"sense_key":"smash%2:30:00::","synset_id":"smash.v.02"},{"index":3,"sense_key":"smash%2:40:00::","synset_id":"bankrupt.v.01"},{"index":4,"sense_key":"smash%2:38:02::","synset_id":"smash.v.04"},{"index":5,"sense_key":"smash%2:37:00::","synset_id":"crush.v.05"},{"index":6,"sense_key":"smash%2:35:08::","synset_id":"bang_up.v.01"},{"index":7,"sense_key":"smash%2:35:06::","synset_id":"smash.v.07"},{"index":8,"sense_key":"smash%2:35:05::","synset_id":"smash.v.08"},{"index":9,"sense_key":"smash%2:33:00::","synset_id":"smash.v.09"},{"index":10,"sense_key":"smash%2:30:10::","synset_id":"smash.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"smash%2:35:05::","call_ids":["senseval3.d000.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"smash%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044909999999999997,"input_uncached_usd":0.0000331,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.273455874994397}
{"item_id":"senseval3.d000.s082.t007","gold_sense_keys":["pavement%1:06:00::","pavement%1:06:01::"],"candidates":[{"index":1,"sense_key":"pavement%1:06:00::","synset_id":"pavement.n.01"},{"index":2,"sense_key":"pavement%1:27:00::","synset_id":"paving.n.01"},{"index":3,"sense_key":"pavement%1:06:01::","synset_id":"sidewalk.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pavement%1:06:01::","call_ids":["senseval3.d000.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pavement%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015255,"input_uncached_usd":0.00001815,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2507541249506176}
{"item_id":"senseval3.d000.s082.t008","gold_sense_keys":["basement%1:06:00::"],"candidates":[{"index":1,"sense_key":"basement%1:06:00::","synset_id":"basement.n.01"},{"index":2,"sense_key":"basement%1:06:01::","synset_id":"basement.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"basement%1:06:00::","call_ids":["senseval3.d000.s082.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"basement%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035584999999999995,"input_uncached_usd":0.000016649999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.429597250011284}
{"item_id":"senseval3.d000.s082.t009","gold_sense_keys":["delivery%1:04:04::"],"candidates":[{"index":1,"sense_key":"delivery%1:04:04::","synset_id":"delivery.n.01"},{"index":2,"sense_key":"delivery%1:11:00::","synset_id":"delivery.n.02"},{"index":3,"sense_key":"delivery%1:10:00::","synset_id":"manner_of_speaking.n.01"},{"index":4,"sense_key":"delivery%1:04:01::","synset_id":"delivery.n.04"},{"index":5,"sense_key":"delivery%1:04:03::","synset_id":"pitch.n.02"},{"index":6,"sense_key":"delivery%1:04:00::","synset_id":"rescue.n.01"},{"index":7,"sense_key":"delivery%1:04:02::","synset_id":"delivery.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"delivery%1:04:04::","call_ids":["senseval3.d000.s082.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"delivery%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036664999999999994,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.936508208978921}
{"item_id":"senseval3.d000.s082.t010","gold_sense_keys":["entrance%1:06:00::"],"candidates":[{"index":1,"sense_key":"entrance%1:06:00::","synset_id":"entrance.n.01"},{"index":2,"sense_key":"entrance%1:11:00::","synset_id":"entrance.n.02"},{"index":3,"sense_key":"entrance%1:04:00::","synset_id":"entrance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"entrance%1:06:00::","call_ids":["senseval3.d000.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"entrance%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048705,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.705826041987166}
{"item_id":"senseval3.d000.s083.t002","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s083.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001068,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.371633499977179}
{"item_id":"senseval3.d000.s083.t005","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d000.s083.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.0000728,"input_uncached_usd":0.000040799999999999996,"input_cached_usd":0.0,"output_usd":0.000032,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.490192333003506}
{"item_id":"senseval3.d000.s083.t006","gold_sense_keys":["blood%1:08:00::"],"candidates":[{"index":1,"sense_key":"blood%1:08:00::","synset_id":"blood.n.01"},{"index":2,"sense_key":"blood%1:07:00::","synset_id":"blood.n.02"},{"index":3,"sense_key":"blood%1:18:00::","synset_id":"rake.n.01"},{"index":4,"sense_key":"blood%1:14:00::","synset_id":"lineage.n.01"},{"index":5,"sense_key":"blood%1:14:01::","synset_id":"blood.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blood%1:08:00::","call_ids":["senseval3.d000.s083.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blood%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015905,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.956259208964184}
{"item_id":"senseval3.d000.s083.t007","gold_sense_keys":["drain%2:38:00::"],"candidates":[{"index":1,"sense_key":"drain%2:38:00::","synset_id":"drain.v.01"},{"index":2,"sense_key":"drain%2:34:00::","synset_id":"drain.v.02"},{"index":3,"sense_key":"drain%2:30:01::","synset_id":"drain.v.03"},{"index":4,"sense_key":"drain%2:30:04::","synset_id":"enfeeble.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drain%2:30:01::","call_ids":["senseval3.d000.s083.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drain%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025745,"input_uncached_usd":0.00002065,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.699052624986507}
{"item_id":"senseval3.d000.s083.t008","gold_sense_keys":["cheek%1:08:00::"],"candidates":[{"index":1,"sense_key":"cheek%1:08:00::","synset_id":"cheek.n.01"},{"index":2,"sense_key":"cheek%1:10:00::","synset_id":"impudence.n.01"},{"index":3,"sense_key":"cheek%1:08:01::","synset_id":"buttock.n.01"},{"index":4,"sense_key":"cheek%1:07:00::","synset_id":"boldness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cheek%1:08:00::","call_ids":["senseval3.d000.s083.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cheek%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010429999999999999,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0732570000109263}
{"item_id":"senseval3.d000.s084.t000","gold_sense_keys":["mutter%2:32:01::"],"candidates":[{"index":1,"sense_key":"mutter%2:32:01::","synset_id":"mumble.v.01"},{"index":2,"sense_key":"mutter%2:32:00::","synset_id":"murmur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mutter%2:32:01::","call_ids":["senseval3.d000.s084.t000__v1__a1","senseval3.d000.s084.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mutter%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":1631,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00069055,"input_uncached_usd":0.00003814999999999999,"input_cached_usd":0.0,"output_usd":0.0006523999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.884742250025738}
{"item_id":"senseval3.d000.s084.t001","gold_sense_keys":["terrible%5:00:00:alarming:00"],"candidates":[{"index":1,"sense_key":"terrible%5:00:00:alarming:00","synset_id":"awful.s.02"},{"index":2,"sense_key":"terrible%5:00:01:bad:00","synset_id":"atrocious.s.02"},{"index":3,"sense_key":"terrible%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":4,"sense_key":"terrible%5:00:00:extraordinary:00","synset_id":"frightful.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"terrible%5:00:00:alarming:00","call_ids":["senseval3.d000.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"terrible%5:00:00:alarming:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":1231,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005169500000000001,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.0004923999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5029546250007115}
{"item_id":"senseval3.d000.s084.t002","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"walk%2:38:02::","call_ids":["senseval3.d000.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"walk%2:38:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004209,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.900287957978435}
{"item_id":"senseval3.d000.s084.t003","gold_sense_keys":["deliberate%5:00:00:unhurried:00"],"candidates":[{"index":1,"sense_key":"deliberate%5:00:00:intended:00","synset_id":"deliberate.s.01"},{"index":2,"sense_key":"deliberate%5:00:00:unhurried:00","synset_id":"careful.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deliberate%5:00:00:unhurried:00","call_ids":["senseval3.d000.s084.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deliberate%5:00:00:unhurried:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015275,"input_uncached_usd":0.00001835,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4889751670416445}
{"item_id":"senseval3.d000.s084.t004","gold_sense_keys":["slowness%1:07:00::"],"candidates":[{"index":1,"sense_key":"slowness%1:09:02::","synset_id":"awkwardness.n.01"},{"index":2,"sense_key":"slowness%1:07:00::","synset_id":"slowness.n.02"},{"index":3,"sense_key":"slowness%1:09:01::","synset_id":"retardation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"slowness%1:07:00::","call_ids":["senseval3.d000.s084.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"slowness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001291,"input_uncached_usd":0.0000203,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4648200410301797}
{"item_id":"senseval3.d000.s084.t005","gold_sense_keys":["elevator%1:06:00::"],"candidates":[{"index":1,"sense_key":"elevator%1:06:00::","synset_id":"elevator.n.01"},{"index":2,"sense_key":"elevator%1:06:01::","synset_id":"elevator.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"elevator%1:06:00::","call_ids":["senseval3.d000.s084.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"elevator%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012644999999999998,"input_uncached_usd":0.00001765,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5079286659602076}
{"item_id":"senseval3.d000.s085.t001","gold_sense_keys":["pour%2:38:04::"],"candidates":[{"index":1,"sense_key":"pour%2:38:03::","synset_id":"pour.v.01"},{"index":2,"sense_key":"pour%2:38:00::","synset_id":"pour.v.02"},{"index":3,"sense_key":"pour%2:38:04::","synset_id":"decant.v.01"},{"index":4,"sense_key":"pour%2:38:02::","synset_id":"pour.v.04"},{"index":5,"sense_key":"pour%2:40:00::","synset_id":"pour.v.05"},{"index":6,"sense_key":"pour%2:43:00::","synset_id":"pour.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"pour%2:38:04::","call_ids":["senseval3.d000.s085.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"pour%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016005,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3246067090076394}
{"item_id":"senseval3.d000.s085.t002","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s085.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026175,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.108178749971557}
{"item_id":"senseval3.d000.s085.t004","gold_sense_keys":["hand%1:08:00::"],"candidates":[{"index":1,"sense_key":"hand%1:08:00::","synset_id":"hand.n.01"},{"index":2,"sense_key":"hand%1:18:00::","synset_id":"hired_hand.n.01"},{"index":3,"sense_key":"hand%1:10:00::","synset_id":"handwriting.n.01"},{"index":4,"sense_key":"hand%1:09:00::","synset_id":"hand.n.04"},{"index":5,"sense_key":"hand%1:15:00::","synset_id":"hand.n.05"},{"index":6,"sense_key":"hand%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"hand%1:09:01::","synset_id":"hand.n.07"},{"index":8,"sense_key":"hand%1:06:00::","synset_id":"hand.n.08"},{"index":9,"sense_key":"hand%1:23:00::","synset_id":"hand.n.09"},{"index":10,"sense_key":"hand%1:18:02::","synset_id":"hand.n.10"},{"index":11,"sense_key":"hand%1:18:01::","synset_id":"bridge_player.n.01"},{"index":12,"sense_key":"hand%1:10:01::","synset_id":"hand.n.12"},{"index":13,"sense_key":"hand%1:05:01::","synset_id":"hand.n.13"},{"index":14,"sense_key":"hand%1:04:00::","synset_id":"hand.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hand%1:08:00::","call_ids":["senseval3.d000.s085.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hand%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002292,"input_uncached_usd":0.000043599999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.799250709009357}
{"item_id":"senseval3.d000.s085.t005","gold_sense_keys":["flop%2:38:01::"],"candidates":[{"index":1,"sense_key":"flop%2:38:01::","synset_id":"flop.v.01"},{"index":2,"sense_key":"flop%2:38:00::","synset_id":"flop.v.02"},{"index":3,"sense_key":"flop%2:41:00::","synset_id":"fall_through.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flop%2:38:01::","call_ids":["senseval3.d000.s085.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flop%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022994999999999998,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.5108592080068775}
{"item_id":"senseval3.d000.s085.t006","gold_sense_keys":["chair%1:06:00::"],"candidates":[{"index":1,"sense_key":"chair%1:06:00::","synset_id":"chair.n.01"},{"index":2,"sense_key":"chair%1:04:00::","synset_id":"professorship.n.01"},{"index":3,"sense_key":"chair%1:18:00::","synset_id":"president.n.04"},{"index":4,"sense_key":"chair%1:06:01::","synset_id":"electric_chair.n.01"},{"index":5,"sense_key":"chair%1:06:02::","synset_id":"chair.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chair%1:06:00::","call_ids":["senseval3.d000.s085.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chair%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002875,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.907260167005006}
{"item_id":"senseval3.d000.s086.t002","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035434999999999997,"input_uncached_usd":0.00004075,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.514303583011497}
{"item_id":"senseval3.d000.s086.t003","gold_sense_keys":["bother%2:41:00::"],"candidates":[{"index":1,"sense_key":"bother%2:41:00::","synset_id":"trouble_oneself.v.01"},{"index":2,"sense_key":"bother%2:37:00::","synset_id":"annoy.v.01"},{"index":3,"sense_key":"bother%2:41:01::","synset_id":"trouble.v.02"},{"index":4,"sense_key":"bother%2:38:00::","synset_id":"bother.v.04"},{"index":5,"sense_key":"bother%2:37:03::","synset_id":"bother.v.05"},{"index":6,"sense_key":"bother%2:37:02::","synset_id":"bother.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bother%2:41:00::","call_ids":["senseval3.d000.s086.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bother%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018764999999999998,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.33089179196395}
{"item_id":"senseval3.d000.s086.t004","gold_sense_keys":["ask%2:32:02::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ask%2:32:02::","call_ids":["senseval3.d000.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ask%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002915,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.166695833031554}
{"item_id":"senseval3.d000.s086.t005","gold_sense_keys":["question%1:10:03::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"question%1:10:00::","call_ids":["senseval3.d000.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"question%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018815,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2822536249877885}
{"item_id":"senseval3.d000.s088.t002","gold_sense_keys":["coincidence%1:11:00::"],"candidates":[{"index":1,"sense_key":"coincidence%1:11:00::","synset_id":"coincidence.n.01"},{"index":2,"sense_key":"coincidence%1:07:02::","synset_id":"coincidence.n.02"},{"index":3,"sense_key":"coincidence%1:07:01::","synset_id":"concurrence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coincidence%1:11:00::","call_ids":["senseval3.d000.s088.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coincidence%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030825,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.756069249997381}
{"item_id":"senseval3.d000.s089.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s089.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00058645,"input_uncached_usd":0.00001645,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.73042166698724}
{"item_id":"senseval3.d000.s090.t001","gold_sense_keys":["happy%3:00:00::"],"candidates":[{"index":1,"sense_key":"happy%3:00:00::","synset_id":"happy.a.01"},{"index":2,"sense_key":"happy%5:00:00:fortunate:00","synset_id":"felicitous.s.01"},{"index":3,"sense_key":"happy%5:00:00:willing:00","synset_id":"glad.s.01"},{"index":4,"sense_key":"happy%5:00:00:felicitous:00","synset_id":"happy.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happy%3:00:00::","call_ids":["senseval3.d000.s090.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happy%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030985,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.204805040964857}
{"item_id":"senseval3.d000.s090.t002","gold_sense_keys":["lousy%5:00:00:nasty:00"],"candidates":[{"index":1,"sense_key":"lousy%5:00:00:bad:00","synset_id":"icky.s.01"},{"index":2,"sense_key":"lousy%5:00:00:dirty:01","synset_id":"lousy.s.02"},{"index":3,"sense_key":"lousy%5:00:00:nasty:00","synset_id":"dirty.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lousy%5:00:00:nasty:00","call_ids":["senseval3.d000.s090.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lousy%5:00:00:nasty:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002565,"input_uncached_usd":0.000019699999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.267322458035778}
{"item_id":"senseval3.d000.s091.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s091.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040474999999999994,"input_uncached_usd":0.000039949999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.726396208978258}
{"item_id":"senseval3.d000.s091.t001","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"have%2:40:04::","call_ids":["senseval3.d000.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"have%2:40:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1098,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001897,"input_uncached_usd":0.0000549,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.973555959004443}
{"item_id":"senseval3.d000.s091.t002","gold_sense_keys":["family%1:14:00::","family%1:14:02::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"family%1:18:00::","call_ids":["senseval3.d000.s091.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"family%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024265,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9490022079553455}
{"item_id":"senseval3.d000.s091.t003","gold_sense_keys":["suffer%2:29:00::","suffer%2:37:00::"],"candidates":[{"index":1,"sense_key":"suffer%2:39:01::","synset_id":"suffer.v.01"},{"index":2,"sense_key":"suffer%2:29:01::","synset_id":"suffer.v.02"},{"index":3,"sense_key":"suffer%2:37:00::","synset_id":"suffer.v.03"},{"index":4,"sense_key":"suffer%2:31:00::","synset_id":"digest.v.03"},{"index":5,"sense_key":"suffer%2:30:00::","synset_id":"suffer.v.05"},{"index":6,"sense_key":"suffer%2:29:00::","synset_id":"suffer.v.06"},{"index":7,"sense_key":"suffer%2:39:00::","synset_id":"hurt.v.06"},{"index":8,"sense_key":"suffer%2:29:03::","synset_id":"suffer.v.08"},{"index":9,"sense_key":"suffer%2:42:04::","synset_id":"suffer.v.09"},{"index":10,"sense_key":"suffer%2:39:02::","synset_id":"suffer.v.10"},{"index":11,"sense_key":"suffer%2:30:02::","synset_id":"suffer.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"suffer%2:31:00::","call_ids":["senseval3.d000.s091.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"suffer%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00070575,"input_uncached_usd":0.00003375,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.746066417021211}
{"item_id":"senseval3.d000.s092.t000","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047829999999999997,"input_uncached_usd":0.0000367,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.6573055410408415}
{"item_id":"senseval3.d000.s092.t001","gold_sense_keys":["kill%2:35:00::"],"candidates":[{"index":1,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":2,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":3,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":4,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":5,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":6,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":7,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":8,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":9,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":10,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":11,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"},{"index":12,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":13,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":14,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":15,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"kill%2:35:00::","call_ids":["senseval3.d000.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"kill%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038054999999999995,"input_uncached_usd":0.000041349999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.737557291984558}
{"item_id":"senseval3.d000.s092.t002","gold_sense_keys":["man%1:18:00::","man%1:18:04::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"man%1:18:04::","call_ids":["senseval3.d000.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"man%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":788,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001738,"input_uncached_usd":0.000039399999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4381457500276156}
{"item_id":"senseval3.d000.s092.t003","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s092.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043684999999999997,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.094528708956204}
{"item_id":"senseval3.d000.s093.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s093.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045749999999999995,"input_uncached_usd":0.0000415,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.7937232090043835}
{"item_id":"senseval3.d000.s093.t001","gold_sense_keys":["mean%2:32:01::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mean%2:32:01::","call_ids":["senseval3.d000.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mean%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034229999999999997,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.531867415993474}
{"item_id":"senseval3.d000.s093.t002","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s093.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007348000000000001,"input_uncached_usd":0.000037199999999999996,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.940543750009965}
{"item_id":"senseval3.d000.s094.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s094.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005094,"input_uncached_usd":0.0000166,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.670843541971408}
{"item_id":"senseval3.d000.s094.t001","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s094.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002349,"input_uncached_usd":0.0000233,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.405093665991444}
{"item_id":"senseval3.d000.s094.t002","gold_sense_keys":["large%3:00:00::"],"candidates":[{"index":1,"sense_key":"large%3:00:00::","synset_id":"large.a.01"},{"index":2,"sense_key":"large%5:00:00:significant:00","synset_id":"large.s.01"},{"index":3,"sense_key":"large%5:00:00:rhetorical:00","synset_id":"bombastic.s.01"},{"index":4,"sense_key":"large%5:00:00:generous:02","synset_id":"big.s.10"},{"index":5,"sense_key":"large%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"large%5:00:00:comprehensive:00","synset_id":"large.s.05"},{"index":7,"sense_key":"large%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"large%3:00:00::","call_ids":["senseval3.d000.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"large%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011369999999999999,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.908607374993153}
{"item_id":"senseval3.d000.s094.t003","gold_sense_keys":["sad%3:00:00::"],"candidates":[{"index":1,"sense_key":"sad%3:00:00::","synset_id":"sad.a.01"},{"index":2,"sense_key":"sad%5:00:00:sorrowful:00","synset_id":"sad.s.01"},{"index":3,"sense_key":"sad%5:00:00:bad:00","synset_id":"deplorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sad%3:00:00::","call_ids":["senseval3.d000.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013049999999999997,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.071461374987848}
{"item_id":"senseval3.d000.s094.t004","gold_sense_keys":["hurt%2:37:00::"],"candidates":[{"index":1,"sense_key":"hurt%2:39:00::","synset_id":"ache.v.03"},{"index":2,"sense_key":"hurt%2:29:01::","synset_id":"hurt.v.02"},{"index":3,"sense_key":"hurt%2:37:01::","synset_id":"pain.v.02"},{"index":4,"sense_key":"hurt%2:30:04::","synset_id":"hurt.v.04"},{"index":5,"sense_key":"hurt%2:37:00::","synset_id":"hurt.v.05"},{"index":6,"sense_key":"hurt%2:39:01::","synset_id":"hurt.v.06"},{"index":7,"sense_key":"hurt%2:29:02::","synset_id":"suffer.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hurt%2:37:00::","call_ids":["senseval3.d000.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hurt%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":914,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039215,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.0003656,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.811916166974697}
{"item_id":"senseval3.d000.s095.t002","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"},{"index":2,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":3,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":4,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":5,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":6,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"dream%1:09:04::","call_ids":["senseval3.d000.s095.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"dream%1:09:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005951000000000001,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.946901207964402}
{"item_id":"senseval3.d000.s095.t003","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"},{"index":2,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":3,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":4,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":5,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":6,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s095.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004927499999999999,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.634352833963931}
{"item_id":"senseval3.d000.s095.t004","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"begin%2:42:04::","call_ids":["senseval3.d000.s095.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"begin%2:42:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":3025,"reasoning_output_tokens":3008},"cost":{"currency":"USD","total_usd":0.00124885,"input_uncached_usd":0.000038849999999999996,"input_cached_usd":0.0,"output_usd":0.00121,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.905678332957905}
{"item_id":"senseval3.d000.s095.t005","gold_sense_keys":["overtone%1:10:00::"],"candidates":[{"index":1,"sense_key":"overtone%1:10:00::","synset_id":"overtone.n.01"},{"index":2,"sense_key":"overtone%1:09:00::","synset_id":"overtone.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"overtone%1:10:00::","call_ids":["senseval3.d000.s095.t005__v1__a1","senseval3.d000.s095.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"overtone%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045884999999999996,"input_uncached_usd":0.00003645,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.019952999951784}
{"item_id":"senseval3.d000.s095.t006","gold_sense_keys":["nightmare%1:09:00::"],"candidates":[{"index":1,"sense_key":"nightmare%1:26:00::","synset_id":"nightmare.n.01"},{"index":2,"sense_key":"nightmare%1:09:00::","synset_id":"nightmare.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nightmare%1:26:00::","call_ids":["senseval3.d000.s095.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nightmare%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":1809,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007395999999999999,"input_uncached_usd":0.000016,"input_cached_usd":0.0,"output_usd":0.0007235999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.938404125045054}
{"item_id":"senseval3.d000.s096.t000","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval3.d000.s096.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037814999999999995,"input_uncached_usd":0.00003895,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.720806541969068}
{"item_id":"senseval3.d000.s096.t001","gold_sense_keys":["dream%1:09:01::"],"candidates":[{"index":1,"sense_key":"dream%1:09:01::","synset_id":"dream.n.01"},{"index":2,"sense_key":"dream%1:09:02::","synset_id":"dream.n.02"},{"index":3,"sense_key":"dream%1:12:00::","synset_id":"ambition.n.01"},{"index":4,"sense_key":"dream%1:09:03::","synset_id":"pipe_dream.n.01"},{"index":5,"sense_key":"dream%1:09:04::","synset_id":"dream.n.05"},{"index":6,"sense_key":"dream%1:26:00::","synset_id":"dream.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dream%1:09:01::","call_ids":["senseval3.d000.s096.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dream%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.00092765,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.754649583017454}
{"item_id":"senseval3.d000.s097.t000","gold_sense_keys":["tight%5:00:00:intoxicated:00"],"candidates":[{"index":1,"sense_key":"tight%3:00:01::","synset_id":"tight.a.01"},{"index":2,"sense_key":"tight%5:00:00:tense:01","synset_id":"taut.s.01"},{"index":3,"sense_key":"tight%5:00:00:invulnerable:00","synset_id":"tight.s.02"},{"index":4,"sense_key":"tight%5:00:00:closed:03","synset_id":"compressed.s.01"},{"index":5,"sense_key":"tight%5:00:00:stingy:00","synset_id":"mean.s.06"},{"index":6,"sense_key":"tight%5:00:00:scarce:00","synset_id":"tight.s.05"},{"index":7,"sense_key":"tight%3:00:02::","synset_id":"tight.a.07"},{"index":8,"sense_key":"tight%5:00:00:fine:00","synset_id":"close.s.06"},{"index":9,"sense_key":"tight%5:00:00:secure:03","synset_id":"tight.s.07"},{"index":10,"sense_key":"tight%5:00:00:equal:00","synset_id":"close.s.03"},{"index":11,"sense_key":"tight%5:00:00:intoxicated:00","synset_id":"besotted.s.01"},{"index":12,"sense_key":"tight%5:00:00:difficult:00","synset_id":"nasty.s.01"},{"index":13,"sense_key":"tight%5:00:00:demanding:01","synset_id":"rigorous.s.02"},{"index":14,"sense_key":"tight%5:00:00:compact:00","synset_id":"tight.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"tight%5:00:00:intoxicated:00","call_ids":["senseval3.d000.s097.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"tight%5:00:00:intoxicated:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":594,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002802,"input_uncached_usd":0.0000426,"input_cached_usd":0.0,"output_usd":0.0002376,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7339776249718852}
{"item_id":"senseval3.d000.s097.t001","gold_sense_keys":["know%2:31:01::","know%2:31:02::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"know%2:31:02::","call_ids":["senseval3.d000.s097.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"know%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":2960,"reasoning_output_tokens":2944},"cost":{"currency":"USD","total_usd":0.0012232,"input_uncached_usd":0.0000392,"input_cached_usd":0.0,"output_usd":0.001184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.530910832982045}
{"item_id":"senseval3.d000.s098.t000","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["senseval3.d000.s098.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023319999999999998,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8239379579899833}
{"item_id":"senseval3.d000.s098.t001","gold_sense_keys":["bar%1:06:04::"],"candidates":[{"index":1,"sense_key":"bar%1:06:04::","synset_id":"barroom.n.01"},{"index":2,"sense_key":"bar%1:06:05::","synset_id":"bar.n.02"},{"index":3,"sense_key":"bar%1:06:00::","synset_id":"bar.n.03"},{"index":4,"sense_key":"bar%1:10:00::","synset_id":"measure.n.07"},{"index":5,"sense_key":"bar%1:06:02::","synset_id":"bar.n.05"},{"index":6,"sense_key":"bar%1:04:00::","synset_id":"prevention.n.01"},{"index":7,"sense_key":"bar%1:23:00::","synset_id":"bar.n.07"},{"index":8,"sense_key":"bar%1:17:00::","synset_id":"bar.n.08"},{"index":9,"sense_key":"bar%1:14:00::","synset_id":"legal_profession.n.01"},{"index":10,"sense_key":"bar%1:07:00::","synset_id":"stripe.n.05"},{"index":11,"sense_key":"bar%1:06:06::","synset_id":"cake.n.01"},{"index":12,"sense_key":"bar%1:06:01::","synset_id":"browning_automatic_rifle.n.01"},{"index":13,"sense_key":"bar%1:06:09::","synset_id":"bar.n.13"},{"index":14,"sense_key":"bar%1:06:08::","synset_id":"bar.n.14"},{"index":15,"sense_key":"bar%1:06:07::","synset_id":"bar.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bar%1:06:04::","call_ids":["senseval3.d000.s098.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bar%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025939999999999996,"input_uncached_usd":0.0000482,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.040086707973387}
{"item_id":"senseval3.d000.s098.t002","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["senseval3.d000.s098.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005121500000000001,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.233621124993078}
{"item_id":"senseval3.d000.s098.t003","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"night%1:28:04::","call_ids":["senseval3.d000.s098.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"night%1:28:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":2321,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009547999999999999,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0009283999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.230802250036504}
{"item_id":"senseval3.d000.s098.t004","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s098.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1114,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004205,"input_uncached_usd":0.0000557,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.898714915965684}
{"item_id":"senseval3.d000.s098.t005","gold_sense_keys":["excuse%1:10:00::"],"candidates":[{"index":1,"sense_key":"excuse%1:10:00::","synset_id":"excuse.n.01"},{"index":2,"sense_key":"excuse%1:10:01::","synset_id":"excuse.n.02"},{"index":3,"sense_key":"excuse%1:09:00::","synset_id":"apology.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"excuse%1:10:00::","call_ids":["senseval3.d000.s098.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"excuse%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002582,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.429464457964059}
{"item_id":"senseval3.d000.s098.t006","gold_sense_keys":["drink%2:34:01::"],"candidates":[{"index":1,"sense_key":"drink%2:34:00::","synset_id":"drink.v.01"},{"index":2,"sense_key":"drink%2:34:01::","synset_id":"drink.v.02"},{"index":3,"sense_key":"drink%2:34:12::","synset_id":"toast.v.02"},{"index":4,"sense_key":"drink%2:31:00::","synset_id":"drink_in.v.01"},{"index":5,"sense_key":"drink%2:34:02::","synset_id":"drink.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drink%2:34:01::","call_ids":["senseval3.d000.s098.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drink%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004152,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.465205457992852}
{"item_id":"senseval3.d000.s098.t007","gold_sense_keys":["usual%3:00:00::"],"candidates":[{"index":1,"sense_key":"usual%3:00:00::","synset_id":"usual.a.01"},{"index":2,"sense_key":"usual%5:00:00:familiar:02","synset_id":"common.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"usual%5:00:00:familiar:02","call_ids":["senseval3.d000.s098.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"usual%5:00:00:familiar:02","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007414,"input_uncached_usd":0.0000182,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.455370542011224}
{"item_id":"senseval3.d000.s099.t000","gold_sense_keys":["sit%2:35:00::","sit%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":3,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":4,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":5,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":6,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":7,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":8,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":9,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":10,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s099.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003986,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.058566291991156}
{"item_id":"senseval3.d000.s099.t001","gold_sense_keys":["usual%3:00:00::"],"candidates":[{"index":1,"sense_key":"usual%3:00:00::","synset_id":"usual.a.01"},{"index":2,"sense_key":"usual%5:00:00:familiar:02","synset_id":"common.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usual%3:00:00::","call_ids":["senseval3.d000.s099.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005116000000000001,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.318365540995728}
{"item_id":"senseval3.d000.s099.t002","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s099.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004504,"input_uncached_usd":0.000034399999999999996,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.2685297499992885}
{"item_id":"senseval3.d000.s099.t003","gold_sense_keys":["little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"little%3:00:01::","call_ids":["senseval3.d000.s099.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"little%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027089999999999997,"input_uncached_usd":0.0000337,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.466732750006486}
{"item_id":"senseval3.d000.s099.t004","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s099.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031485,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.074220167007297}
{"item_id":"senseval3.d000.s099.t006","gold_sense_keys":["talkative%5:00:00:communicative:00","talkative%5:00:00:voluble:00"],"candidates":[{"index":1,"sense_key":"talkative%5:00:00:voluble:00","synset_id":"chatty.s.01"},{"index":2,"sense_key":"talkative%5:00:00:indiscreet:00","synset_id":"bigmouthed.s.01"},{"index":3,"sense_key":"talkative%5:00:00:communicative:00","synset_id":"expansive.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talkative%5:00:00:voluble:00","call_ids":["senseval3.d000.s099.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talkative%5:00:00:voluble:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033549999999999997,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.321797624987084}
{"item_id":"senseval3.d000.s099.t007","gold_sense_keys":["friendly%3:00:01::"],"candidates":[{"index":1,"sense_key":"friendly%3:00:01::","synset_id":"friendly.a.01"},{"index":2,"sense_key":"friendly%5:00:00:amicable:00","synset_id":"friendly.s.01"},{"index":3,"sense_key":"friendly%3:00:03::","synset_id":"friendly.a.03"},{"index":4,"sense_key":"friendly%3:00:02::","synset_id":"friendly.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"friendly%5:00:00:amicable:00","call_ids":["senseval3.d000.s099.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"friendly%5:00:00:amicable:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000285,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.964308583003003}
{"item_id":"senseval3.d000.s100.t000","gold_sense_keys":["enjoy%2:34:00::","enjoy%2:37:00::","enjoy%2:37:01::"],"candidates":[{"index":1,"sense_key":"enjoy%2:37:00::","synset_id":"enjoy.v.01"},{"index":2,"sense_key":"enjoy%2:34:01::","synset_id":"enjoy.v.02"},{"index":3,"sense_key":"enjoy%2:37:01::","synset_id":"love.v.02"},{"index":4,"sense_key":"enjoy%2:39:00::","synset_id":"enjoy.v.04"},{"index":5,"sense_key":"enjoy%2:34:00::","synset_id":"delight.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"enjoy%2:37:00::","call_ids":["senseval3.d000.s100.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"enjoy%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000285,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.49941933300579}
{"item_id":"senseval3.d000.s100.t002","gold_sense_keys":["ritual%1:04:00::"],"candidates":[{"index":1,"sense_key":"ritual%1:04:00::","synset_id":"ritual.n.01"},{"index":2,"sense_key":"ritual%1:04:01::","synset_id":"ritual.n.02"},{"index":3,"sense_key":"ritual%1:04:02::","synset_id":"ritual.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ritual%1:04:02::","call_ids":["senseval3.d000.s100.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ritual%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017774999999999998,"input_uncached_usd":0.000017749999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.719210458046291}
{"item_id":"senseval3.d000.s100.t003","gold_sense_keys":["wash_down%2:34:00::"],"candidates":[{"index":1,"sense_key":"wash_down%2:34:00::","synset_id":"wash_down.v.01"},{"index":2,"sense_key":"wash_down%2:30:00::","synset_id":"wash_down.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wash_down%2:34:00::","call_ids":["senseval3.d000.s100.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wash_down%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004089,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.448925958015025}
{"item_id":"senseval3.d000.s100.t004","gold_sense_keys":["pet%1:18:00::"],"candidates":[{"index":1,"sense_key":"pet%1:05:00::","synset_id":"pet.n.01"},{"index":2,"sense_key":"pet%1:18:00::","synset_id":"darling.n.01"},{"index":3,"sense_key":"pet%1:12:00::","synset_id":"pet.n.03"},{"index":4,"sense_key":"pet%1:04:00::","synset_id":"positron_emission_tomography.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pet%1:18:00::","call_ids":["senseval3.d000.s100.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pet%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053945,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.303557792038191}
{"item_id":"senseval3.d000.s100.t005","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":2,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"},{"index":3,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"grievance%1:10:00::","call_ids":["senseval3.d000.s100.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"grievance%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020534999999999998,"input_uncached_usd":0.00001975,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.790431040979456}
{"item_id":"senseval3.d000.s100.t006","gold_sense_keys":["bourbon%1:13:00::"],"candidates":[{"index":1,"sense_key":"bourbon%1:18:01::","synset_id":"bourbon.n.01"},{"index":2,"sense_key":"bourbon%1:13:00::","synset_id":"bourbon.n.02"},{"index":3,"sense_key":"bourbon%1:18:00::","synset_id":"bourbon.n.03"},{"index":4,"sense_key":"bourbon%1:14:00::","synset_id":"bourbon.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bourbon%1:13:00::","call_ids":["senseval3.d000.s100.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bourbon%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018094999999999998,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.190159290970769}
{"item_id":"senseval3.d000.s100.t007","gold_sense_keys":["water%1:13:00::"],"candidates":[{"index":1,"sense_key":"water%1:27:00::","synset_id":"water.n.01"},{"index":2,"sense_key":"water%1:17:00::","synset_id":"body_of_water.n.01"},{"index":3,"sense_key":"water%1:27:02::","synset_id":"water.n.03"},{"index":4,"sense_key":"water%1:06:00::","synset_id":"water_system.n.02"},{"index":5,"sense_key":"water%1:27:01::","synset_id":"urine.n.01"},{"index":6,"sense_key":"water%1:13:00::","synset_id":"water.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"water%1:27:00::","call_ids":["senseval3.d000.s100.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"water%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000317,"input_uncached_usd":0.000029,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7696646669646725}
{"item_id":"senseval3.d000.s101.t000","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d000.s101.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061305,"input_uncached_usd":0.00001745,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.327574124967214}
{"item_id":"senseval3.d000.s101.t001","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":2,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"},{"index":3,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"grievance%1:10:00::","call_ids":["senseval3.d000.s101.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"grievance%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030695,"input_uncached_usd":0.00001895,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.968484792043455}
{"item_id":"senseval3.d000.s102.t000","gold_sense_keys":["forget%2:31:01::"],"candidates":[{"index":1,"sense_key":"forget%2:31:01::","synset_id":"forget.v.01"},{"index":2,"sense_key":"forget%2:31:00::","synset_id":"forget.v.02"},{"index":3,"sense_key":"forget%2:31:03::","synset_id":"forget.v.03"},{"index":4,"sense_key":"forget%2:31:02::","synset_id":"forget.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"forget%2:31:01::","call_ids":["senseval3.d000.s102.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"forget%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020784999999999998,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9387544579803944}
{"item_id":"senseval3.d000.s103.t000","gold_sense_keys":["mention%2:32:02::"],"candidates":[{"index":1,"sense_key":"mention%2:32:02::","synset_id":"mention.v.01"},{"index":2,"sense_key":"mention%2:32:00::","synset_id":"note.v.01"},{"index":3,"sense_key":"mention%2:32:01::","synset_id":"mention.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mention%2:32:02::","call_ids":["senseval3.d000.s103.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mention%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005635,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.343044457957149}
{"item_id":"senseval3.d000.s104.t000","gold_sense_keys":["real%3:00:02::","real%5:00:00:true:00"],"candidates":[{"index":1,"sense_key":"real%3:00:00::","synset_id":"real.a.01"},{"index":2,"sense_key":"real%3:00:02::","synset_id":"real.a.02"},{"index":3,"sense_key":"real%5:00:00:serious:00","synset_id":"real.s.01"},{"index":4,"sense_key":"real%5:00:00:concrete:00","synset_id":"real.s.02"},{"index":5,"sense_key":"real%5:00:00:true:00","synset_id":"actual.s.02"},{"index":6,"sense_key":"real%3:00:01::","synset_id":"real.a.06"},{"index":7,"sense_key":"real%3:00:03::","synset_id":"substantial.a.03"},{"index":8,"sense_key":"real%5:00:00:tangible:02","synset_id":"real.s.04"},{"index":9,"sense_key":"real%5:00:02:realistic:00","synset_id":"veridical.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"real%5:00:00:true:00","call_ids":["senseval3.d000.s104.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"real%5:00:00:true:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005284500000000001,"input_uncached_usd":0.00003565,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.523228332982399}
{"item_id":"senseval3.d000.s104.t001","gold_sense_keys":["grievance%1:12:00::"],"candidates":[{"index":1,"sense_key":"grievance%1:12:00::","synset_id":"grudge.n.01"},{"index":2,"sense_key":"grievance%1:10:01::","synset_id":"grievance.n.02"},{"index":3,"sense_key":"grievance%1:10:00::","synset_id":"grievance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"grievance%1:12:00::","call_ids":["senseval3.d000.s104.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"grievance%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040985,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.835945250000805}
{"item_id":"senseval3.d000.s105.t000","gold_sense_keys":["end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["senseval3.d000.s105.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005108,"input_uncached_usd":0.000043599999999999996,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.797792042023502}
{"item_id":"senseval3.d000.s105.t003","gold_sense_keys":["make%2:36:00::","make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval3.d000.s105.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2235,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006813500000000001,"input_uncached_usd":0.00011174999999999999,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.912653084029444}
{"item_id":"senseval3.d000.s105.t004","gold_sense_keys":["wet%3:00:01::"],"candidates":[{"index":1,"sense_key":"wet%3:00:01::","synset_id":"wet.a.01"},{"index":2,"sense_key":"wet%3:00:05::","synset_id":"wet.a.02"},{"index":3,"sense_key":"wet%3:00:04::","synset_id":"wet.a.03"},{"index":4,"sense_key":"wet%3:00:02::","synset_id":"wet.a.04"},{"index":5,"sense_key":"wet%5:00:00:alcoholic:00","synset_id":"wet.s.01"},{"index":6,"sense_key":"wet%5:00:00:intoxicated:00","synset_id":"besotted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wet%3:00:01::","call_ids":["senseval3.d000.s105.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wet%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018529999999999998,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.892717500042636}
{"item_id":"senseval3.d000.s105.t005","gold_sense_keys":["ring%1:25:00::"],"candidates":[{"index":1,"sense_key":"ring%1:07:00::","synset_id":"ring.n.01"},{"index":2,"sense_key":"ring%1:25:00::","synset_id":"ring.n.02"},{"index":3,"sense_key":"ring%1:06:02::","synset_id":"hoop.n.02"},{"index":4,"sense_key":"ring%1:17:00::","synset_id":"closed_chain.n.01"},{"index":5,"sense_key":"ring%1:14:00::","synset_id":"gang.n.01"},{"index":6,"sense_key":"ring%1:11:00::","synset_id":"ring.n.06"},{"index":7,"sense_key":"ring%1:06:03::","synset_id":"ring.n.07"},{"index":8,"sense_key":"ring%1:06:00::","synset_id":"ring.n.08"},{"index":9,"sense_key":"ring%1:06:01::","synset_id":"band.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ring%1:25:00::","call_ids":["senseval3.d000.s105.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ring%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005523500000000001,"input_uncached_usd":0.00003395,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.132350000028964}
{"item_id":"senseval3.d000.s105.t006","gold_sense_keys":["bottom%1:15:00::"],"candidates":[{"index":1,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":2,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":3,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"},{"index":4,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":7,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bottom%1:15:01::","call_ids":["senseval3.d000.s105.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bottom%1:15:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021385,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.773290166980587}
{"item_id":"senseval3.d000.s105.t007","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"glass%1:23:00::","call_ids":["senseval3.d000.s105.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"glass%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007239499999999999,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.015117583039682}
{"item_id":"senseval3.d000.s105.t008","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d000.s105.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038695,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.961631166981533}
{"item_id":"senseval3.d000.s105.t009","gold_sense_keys":["aware%3:00:00::"],"candidates":[{"index":1,"sense_key":"aware%3:00:00::","synset_id":"aware.a.01"},{"index":2,"sense_key":"aware%3:00:04::","synset_id":"mindful.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aware%3:00:00::","call_ids":["senseval3.d000.s105.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aware%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035844999999999996,"input_uncached_usd":0.00001885,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.75684312498197}
{"item_id":"senseval3.d000.s106.t000","gold_sense_keys":["look%2:39:00::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"look%2:39:00::","call_ids":["senseval3.d000.s106.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"look%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032405,"input_uncached_usd":0.000036049999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.588103833957575}
{"item_id":"senseval3.d000.s106.t002","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s106.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003117,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.793543167004827}
{"item_id":"senseval3.d000.s106.t004","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s106.t004__v1__a1","senseval3.d000.s106.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":1952,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008171499999999999,"input_uncached_usd":0.00003635,"input_cached_usd":0.0,"output_usd":0.0007808,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.584228417021222}
{"item_id":"senseval3.d000.s106.t005","gold_sense_keys":["sit%2:35:00::"],"candidates":[{"index":1,"sense_key":"sit%2:35:00::","synset_id":"sit.v.01"},{"index":2,"sense_key":"sit%2:42:00::","synset_id":"sit.v.02"},{"index":3,"sense_key":"sit%2:38:00::","synset_id":"sit_down.v.01"},{"index":4,"sense_key":"sit%2:38:01::","synset_id":"sit.v.04"},{"index":5,"sense_key":"sit%2:39:00::","synset_id":"model.v.03"},{"index":6,"sense_key":"sit%2:38:03::","synset_id":"ride.v.01"},{"index":7,"sense_key":"sit%2:42:05::","synset_id":"sit.v.07"},{"index":8,"sense_key":"sit%2:41:05::","synset_id":"baby-sit.v.02"},{"index":9,"sense_key":"sit%2:35:01::","synset_id":"seat.v.01"},{"index":10,"sense_key":"sit%2:33:00::","synset_id":"sit.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit%2:35:00::","call_ids":["senseval3.d000.s106.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002956,"input_uncached_usd":0.0000332,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.049365332990419}
{"item_id":"senseval3.d000.s106.t006","gold_sense_keys":["table%1:06:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d000.s106.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044225,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.631997958000284}
{"item_id":"senseval3.d000.s106.t008","gold_sense_keys":["secret%5:00:03:covert:00"],"candidates":[{"index":1,"sense_key":"secret%5:00:00:concealed:00","synset_id":"secret.s.01"},{"index":2,"sense_key":"secret%5:00:01:covert:00","synset_id":"clandestine.s.01"},{"index":3,"sense_key":"secret%5:00:00:unacknowledged:00","synset_id":"unavowed.s.03"},{"index":4,"sense_key":"secret%5:00:03:covert:00","synset_id":"secret.s.04"},{"index":5,"sense_key":"secret%5:00:00:inward:00","synset_id":"secret.s.05"},{"index":6,"sense_key":"secret%5:00:02:concealed:00","synset_id":"hidden.s.02"},{"index":7,"sense_key":"secret%5:00:02:private:00","synset_id":"privy.s.01"},{"index":8,"sense_key":"secret%5:00:00:private:00","synset_id":"confidential.s.02"},{"index":9,"sense_key":"secret%5:00:02:covert:00","synset_id":"secret.s.09"},{"index":10,"sense_key":"secret%5:00:00:esoteric:00","synset_id":"mysterious.s.02"},{"index":11,"sense_key":"secret%5:00:00:classified:02","synset_id":"secret.s.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"secret%5:00:00:concealed:00","call_ids":["senseval3.d000.s106.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"secret%5:00:00:concealed:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005575,"input_uncached_usd":0.000039099999999999995,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.920604458020534}
{"item_id":"senseval3.d000.s107.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s107.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.000338,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.689381790987682}
{"item_id":"senseval3.d000.s107.t001","gold_sense_keys":["name%1:10:00::"],"candidates":[{"index":1,"sense_key":"name%1:10:00::","synset_id":"name.n.01"},{"index":2,"sense_key":"name%1:26:00::","synset_id":"name.n.02"},{"index":3,"sense_key":"name%1:14:00::","synset_id":"name.n.03"},{"index":4,"sense_key":"name%1:18:00::","synset_id":"name.n.04"},{"index":5,"sense_key":"name%1:04:00::","synset_id":"name.n.05"},{"index":6,"sense_key":"name%1:10:01::","synset_id":"name.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"name%1:10:00::","call_ids":["senseval3.d000.s107.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"name%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010874999999999999,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5654507920262404}
{"item_id":"senseval3.d000.s107.t002","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d000.s107.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1431,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00020635,"input_uncached_usd":0.00007154999999999999,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.166491625015624}
{"item_id":"senseval3.d000.s107.t003","gold_sense_keys":["sit_down%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit_down%2:38:00::","synset_id":"sit_down.v.01"},{"index":2,"sense_key":"sit_down%2:35:00::","synset_id":"seat.v.01"},{"index":3,"sense_key":"sit_down%2:35:03::","synset_id":"sit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit_down%2:38:00::","call_ids":["senseval3.d000.s107.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit_down%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005631,"input_uncached_usd":0.0000191,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.780227167007979}
{"item_id":"senseval3.d000.s108.t000","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s108.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040689999999999997,"input_uncached_usd":0.0000421,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.752600917010568}
{"item_id":"senseval3.d000.s108.t001","gold_sense_keys":["uncanny%5:00:00:supernatural:00"],"candidates":[{"index":1,"sense_key":"uncanny%5:00:00:supernatural:00","synset_id":"eldritch.s.01"},{"index":2,"sense_key":"uncanny%5:00:00:extraordinary:00","synset_id":"preternatural.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"uncanny%5:00:00:extraordinary:00","call_ids":["senseval3.d000.s108.t001__v1__a1","senseval3.d000.s108.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"uncanny%5:00:00:extraordinary:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007499500000000001,"input_uncached_usd":0.00004594999999999999,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.459406416979618}
{"item_id":"senseval3.d000.s108.t002","gold_sense_keys":["shadow%1:09:00::","shadow%1:26:01::"],"candidates":[{"index":1,"sense_key":"shadow%1:26:01::","synset_id":"shadow.n.01"},{"index":2,"sense_key":"shadow%1:15:00::","synset_id":"darkness.n.02"},{"index":3,"sense_key":"shadow%1:09:00::","synset_id":"apparition.n.03"},{"index":4,"sense_key":"shadow%1:12:00::","synset_id":"shadow.n.04"},{"index":5,"sense_key":"shadow%1:10:00::","synset_id":"trace.n.02"},{"index":6,"sense_key":"shadow%1:07:00::","synset_id":"shadow.n.06"},{"index":7,"sense_key":"shadow%1:26:00::","synset_id":"shadow.n.07"},{"index":8,"sense_key":"shadow%1:18:00::","synset_id":"tail.n.05"},{"index":9,"sense_key":"shadow%1:18:01::","synset_id":"shadow.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shadow%1:09:00::","call_ids":["senseval3.d000.s108.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shadow%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037104999999999994,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.668918540992308}
{"item_id":"senseval3.d000.s108.t003","gold_sense_keys":["make%2:36:00::","make%2:36:08::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"make%2:36:00::","call_ids":["senseval3.d000.s108.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"make%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2262,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0007083,"input_uncached_usd":0.0001131,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.956830625014845}
{"item_id":"senseval3.d000.s108.t005","gold_sense_keys":["noise%1:11:00::"],"candidates":[{"index":1,"sense_key":"noise%1:11:00::","synset_id":"noise.n.01"},{"index":2,"sense_key":"noise%1:09:00::","synset_id":"noise.n.02"},{"index":3,"sense_key":"noise%1:11:01::","synset_id":"noise.n.03"},{"index":4,"sense_key":"noise%1:10:00::","synset_id":"noise.n.04"},{"index":5,"sense_key":"noise%1:07:01::","synset_id":"noise.n.05"},{"index":6,"sense_key":"noise%1:07:00::","synset_id":"randomness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"noise%1:11:00::","call_ids":["senseval3.d000.s108.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"noise%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057345,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.40698391594924}
{"item_id":"senseval3.d000.s108.t006","gold_sense_keys":["shadow%1:09:00::","shadow%1:26:01::"],"candidates":[{"index":1,"sense_key":"shadow%1:26:01::","synset_id":"shadow.n.01"},{"index":2,"sense_key":"shadow%1:15:00::","synset_id":"darkness.n.02"},{"index":3,"sense_key":"shadow%1:09:00::","synset_id":"apparition.n.03"},{"index":4,"sense_key":"shadow%1:12:00::","synset_id":"shadow.n.04"},{"index":5,"sense_key":"shadow%1:10:00::","synset_id":"trace.n.02"},{"index":6,"sense_key":"shadow%1:07:00::","synset_id":"shadow.n.06"},{"index":7,"sense_key":"shadow%1:26:00::","synset_id":"shadow.n.07"},{"index":8,"sense_key":"shadow%1:18:00::","synset_id":"tail.n.05"},{"index":9,"sense_key":"shadow%1:18:01::","synset_id":"shadow.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shadow%1:09:00::","call_ids":["senseval3.d000.s108.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shadow%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007290500000000001,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.789319916977547}
{"item_id":"senseval3.d000.s109.t001","gold_sense_keys":["shrink%2:30:00::","shrink%2:38:00::"],"candidates":[{"index":1,"sense_key":"shrink%2:30:02::","synset_id":"shrivel.v.01"},{"index":2,"sense_key":"shrink%2:38:00::","synset_id":"flinch.v.01"},{"index":3,"sense_key":"shrink%2:30:01::","synset_id":"shrink.v.03"},{"index":4,"sense_key":"shrink%2:30:00::","synset_id":"shrink.v.04"},{"index":5,"sense_key":"shrink%2:30:11::","synset_id":"shrink.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shrink%2:38:00::","call_ids":["senseval3.d000.s109.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shrink%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023884999999999998,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.670537209021859}
{"item_id":"senseval3.d000.s109.t003","gold_sense_keys":["trap%2:35:00::","trap%2:35:02::","trap%2:41:00::"],"candidates":[{"index":1,"sense_key":"trap%2:41:00::","synset_id":"trap.v.01"},{"index":2,"sense_key":"trap%2:35:00::","synset_id":"trap.v.02"},{"index":3,"sense_key":"trap%2:35:01::","synset_id":"trap.v.03"},{"index":4,"sense_key":"trap%2:35:02::","synset_id":"trap.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trap%2:41:00::","call_ids":["senseval3.d000.s109.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trap%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033765,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.246855666046031}
{"item_id":"senseval3.d000.s109.t004","gold_sense_keys":["corner%1:06:02::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"corner%1:06:02::","call_ids":["senseval3.d000.s109.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"corner%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005798499999999999,"input_uncached_usd":0.00003545,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.783601249975618}
{"item_id":"senseval3.d000.s109.t005","gold_sense_keys":["wiry%5:00:00:thin:03"],"candidates":[{"index":1,"sense_key":"wiry%5:00:00:thin:03","synset_id":"stringy.s.01"},{"index":2,"sense_key":"wiry%3:01:00::","synset_id":"wiry.a.02"},{"index":3,"sense_key":"wiry%5:00:00:hairy:00","synset_id":"wiry.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wiry%5:00:00:thin:03","call_ids":["senseval3.d000.s109.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wiry%5:00:00:thin:03","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015414999999999998,"input_uncached_usd":0.00001975,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0272356249624863}
{"item_id":"senseval3.d000.s109.t007","gold_sense_keys":["little%3:00:00::","little%3:00:01::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"little%3:00:00::","call_ids":["senseval3.d000.s109.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"little%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014355,"input_uncached_usd":0.00003475,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.598788915958721}
{"item_id":"senseval3.d000.s109.t008","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s109.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":846,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002535,"input_uncached_usd":0.0000423,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.444689791998826}
{"item_id":"senseval3.d000.s110.t000","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d000.s110.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":831,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032955,"input_uncached_usd":0.00004155,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.953153417038266}
{"item_id":"senseval3.d000.s110.t001","gold_sense_keys":["wish%2:32:02::","wish%2:37:00::"],"candidates":[{"index":1,"sense_key":"wish%2:37:00::","synset_id":"wish.v.01"},{"index":2,"sense_key":"wish%2:37:02::","synset_id":"wish.v.02"},{"index":3,"sense_key":"wish%2:32:02::","synset_id":"wish.v.03"},{"index":4,"sense_key":"wish%2:37:01::","synset_id":"wish.v.04"},{"index":5,"sense_key":"wish%2:32:01::","synset_id":"wish.v.05"},{"index":6,"sense_key":"wish%2:32:00::","synset_id":"wish.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wish%2:32:02::","call_ids":["senseval3.d000.s110.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wish%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026325,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.098156291991472}
{"item_id":"senseval3.d000.s110.t002","gold_sense_keys":["shout%2:32:01::"],"candidates":[{"index":1,"sense_key":"shout%2:32:00::","synset_id":"shout.v.01"},{"index":2,"sense_key":"shout%2:32:01::","synset_id":"shout.v.02"},{"index":3,"sense_key":"shout%2:32:08::","synset_id":"exclaim.v.01"},{"index":4,"sense_key":"shout%2:32:12::","synset_id":"abuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shout%2:32:01::","call_ids":["senseval3.d000.s110.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shout%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005208000000000001,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.638216833001934}
{"item_id":"senseval3.d000.s110.t003","gold_sense_keys":["other%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%5:00:00:past:00","call_ids":["senseval3.d000.s110.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004919999999999999,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.017745500023011}
{"item_id":"senseval3.d000.s110.t004","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval3.d000.s110.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051555,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.537323208001908}
{"item_id":"senseval3.d000.s110.t005","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d000.s110.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005129500000000001,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.96859470900381}
{"item_id":"senseval3.d000.s110.t006","gold_sense_keys":["bear_down%2:33:03::"],"candidates":[{"index":1,"sense_key":"bear_down%2:35:00::","synset_id":"drag_down.v.01"},{"index":2,"sense_key":"bear_down%2:35:01::","synset_id":"bear_down.v.02"},{"index":3,"sense_key":"bear_down%2:33:03::","synset_id":"charge.v.01"},{"index":4,"sense_key":"bear_down%2:33:00::","synset_id":"bear_down.v.04"},{"index":5,"sense_key":"bear_down%2:32:12::","synset_id":"bear_down.v.05"},{"index":6,"sense_key":"bear_down%2:32:01::","synset_id":"bear_down.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"bear_down%2:33:00::","call_ids":["senseval3.d000.s110.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"bear_down%2:33:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006742499999999999,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.536869332951028}
{"item_id":"senseval3.d000.s110.t007","gold_sense_keys":["crossing%1:06:00::","crossing%1:06:01::"],"candidates":[{"index":1,"sense_key":"crossing%1:04:01::","synset_id":"crossing.n.01"},{"index":2,"sense_key":"crossing%1:17:00::","synset_id":"ford.n.07"},{"index":3,"sense_key":"crossing%1:15:00::","synset_id":"crossing.n.03"},{"index":4,"sense_key":"crossing%1:06:01::","synset_id":"intersection.n.02"},{"index":5,"sense_key":"crossing%1:06:00::","synset_id":"crossing.n.05"},{"index":6,"sense_key":"crossing%1:04:02::","synset_id":"hybridization.n.01"},{"index":7,"sense_key":"crossing%1:04:00::","synset_id":"crossing.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"crossing%1:06:01::","call_ids":["senseval3.d000.s110.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"crossing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021475,"input_uncached_usd":0.000029149999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.17160204204265}
{"item_id":"senseval3.d000.s111.t000","gold_sense_keys":["saddle%2:35:10::"],"candidates":[{"index":1,"sense_key":"saddle%2:35:01::","synset_id":"saddle.v.01"},{"index":2,"sense_key":"saddle%2:35:10::","synset_id":"saddle.v.02"},{"index":3,"sense_key":"saddle%2:32:00::","synset_id":"charge.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"saddle%2:35:10::","call_ids":["senseval3.d000.s111.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"saddle%2:35:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056435,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.755731375014875}
{"item_id":"senseval3.d000.s111.t001","gold_sense_keys":["creep%1:18:00::"],"candidates":[{"index":1,"sense_key":"creep%1:18:00::","synset_id":"creep.n.01"},{"index":2,"sense_key":"creep%1:11:00::","synset_id":"creep.n.02"},{"index":3,"sense_key":"creep%1:06:00::","synset_id":"creep.n.03"},{"index":4,"sense_key":"creep%1:04:00::","synset_id":"crawl.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"creep%1:18:00::","call_ids":["senseval3.d000.s111.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"creep%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020805,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4632979589514434}
{"item_id":"senseval3.d000.s112.t000","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval3.d000.s112.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008461,"input_uncached_usd":0.000046099999999999996,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.208127124991734}
{"item_id":"senseval3.d000.s112.t001","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"go%2:38:09::","call_ids":["senseval3.d000.s112.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"go%2:38:09::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1693,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00037265,"input_uncached_usd":0.00008465,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.266163124993909}
{"item_id":"senseval3.d000.s112.t002","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"other%5:00:00:past:00","call_ids":["senseval3.d000.s112.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"other%5:00:00:past:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041385,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.790472209046129}
{"item_id":"senseval3.d000.s112.t003","gold_sense_keys":["place%1:15:04::"],"candidates":[{"index":1,"sense_key":"place%1:15:00::","synset_id":"topographic_point.n.01"},{"index":2,"sense_key":"place%1:15:04::","synset_id":"place.n.02"},{"index":3,"sense_key":"place%1:09:00::","synset_id":"place.n.03"},{"index":4,"sense_key":"place%1:15:01::","synset_id":"place.n.04"},{"index":5,"sense_key":"place%1:04:01::","synset_id":"stead.n.01"},{"index":6,"sense_key":"place%1:26:01::","synset_id":"place.n.06"},{"index":7,"sense_key":"place%1:15:06::","synset_id":"home.n.01"},{"index":8,"sense_key":"place%1:04:00::","synset_id":"position.n.06"},{"index":9,"sense_key":"place%1:15:05::","synset_id":"position.n.01"},{"index":10,"sense_key":"place%1:26:00::","synset_id":"place.n.10"},{"index":11,"sense_key":"place%1:15:03::","synset_id":"seat.n.01"},{"index":12,"sense_key":"place%1:10:00::","synset_id":"place.n.12"},{"index":13,"sense_key":"place%1:26:02::","synset_id":"place.n.13"},{"index":14,"sense_key":"place%1:15:02::","synset_id":"plaza.n.01"},{"index":15,"sense_key":"place%1:10:02::","synset_id":"place.n.15"},{"index":16,"sense_key":"place%1:10:01::","synset_id":"space.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"place%1:15:00::","call_ids":["senseval3.d000.s112.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"place%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":971,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00054135,"input_uncached_usd":0.00004855,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.789578958007041}
{"item_id":"senseval3.d000.s113.t000","gold_sense_keys":["low%5:00:00:soft:04"],"candidates":[{"index":1,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":2,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":3,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":4,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":5,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":6,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":7,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":8,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"},{"index":9,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":10,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"low%5:00:00:soft:04","call_ids":["senseval3.d000.s113.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"low%5:00:00:soft:04","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027465,"input_uncached_usd":0.000037449999999999995,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.835123041004408}
{"item_id":"senseval3.d000.s113.t001","gold_sense_keys":["voice%1:07:00::"],"candidates":[{"index":1,"sense_key":"voice%1:07:00::","synset_id":"voice.n.01"},{"index":2,"sense_key":"voice%1:10:00::","synset_id":"voice.n.02"},{"index":3,"sense_key":"voice%1:11:00::","synset_id":"voice.n.03"},{"index":4,"sense_key":"voice%1:10:01::","synset_id":"articulation.n.03"},{"index":5,"sense_key":"voice%1:04:00::","synset_id":"voice.n.05"},{"index":6,"sense_key":"voice%1:10:03::","synset_id":"voice.n.06"},{"index":7,"sense_key":"voice%1:18:01::","synset_id":"voice.n.07"},{"index":8,"sense_key":"voice%1:18:00::","synset_id":"spokesperson.n.01"},{"index":9,"sense_key":"voice%1:07:01::","synset_id":"voice.n.09"},{"index":10,"sense_key":"voice%1:24:00::","synset_id":"voice.n.10"},{"index":11,"sense_key":"voice%1:10:02::","synset_id":"part.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"voice%1:07:00::","call_ids":["senseval3.d000.s113.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"voice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035269999999999995,"input_uncached_usd":0.000039099999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.952622874989174}
{"item_id":"senseval3.d000.s113.t003","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s113.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032995,"input_uncached_usd":0.000041949999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.476391583040822}
{"item_id":"senseval3.d000.s113.t004","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ask%2:32:04::","call_ids":["senseval3.d000.s113.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ask%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004979,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1500950830522925}
{"item_id":"senseval3.d000.s113.t005","gold_sense_keys":["happy%3:00:00::"],"candidates":[{"index":1,"sense_key":"happy%3:00:00::","synset_id":"happy.a.01"},{"index":2,"sense_key":"happy%5:00:00:fortunate:00","synset_id":"felicitous.s.01"},{"index":3,"sense_key":"happy%5:00:00:willing:00","synset_id":"glad.s.01"},{"index":4,"sense_key":"happy%5:00:00:felicitous:00","synset_id":"happy.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"happy%5:00:00:willing:00","call_ids":["senseval3.d000.s113.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"happy%5:00:00:willing:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036279999999999993,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.950312500004657}
{"item_id":"senseval3.d000.s113.t006","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s113.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030104999999999997,"input_uncached_usd":0.00003865,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.705056417034939}
{"item_id":"senseval3.d000.s114.t000","gold_sense_keys":["make%2:41:13::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":24,"chosen_sense_key":"make%2:41:13::","call_ids":["senseval3.d000.s114.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":24,"predicted_sense_key":"make%2:41:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2252,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002982,"input_uncached_usd":0.0001126,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.664709791948553}
{"item_id":"senseval3.d000.s114.t001","gold_sense_keys":["mistake%1:04:00::"],"candidates":[{"index":1,"sense_key":"mistake%1:04:00::","synset_id":"mistake.n.01"},{"index":2,"sense_key":"mistake%1:09:00::","synset_id":"mistake.n.02"},{"index":3,"sense_key":"mistake%1:10:00::","synset_id":"error.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mistake%1:04:00::","call_ids":["senseval3.d000.s114.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mistake%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018204999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.350852500007022}
{"item_id":"senseval3.d000.s114.t002","gold_sense_keys":["answer%2:32:00::"],"candidates":[{"index":1,"sense_key":"answer%2:32:00::","synset_id":"answer.v.01"},{"index":2,"sense_key":"answer%2:31:02::","synset_id":"answer.v.02"},{"index":3,"sense_key":"answer%2:31:00::","synset_id":"answer.v.03"},{"index":4,"sense_key":"answer%2:31:03::","synset_id":"answer.v.04"},{"index":5,"sense_key":"answer%2:32:06::","synset_id":"answer.v.05"},{"index":6,"sense_key":"answer%2:42:01::","synset_id":"answer.v.06"},{"index":7,"sense_key":"answer%2:42:02::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"answer%2:42:00::","synset_id":"answer.v.08"},{"index":9,"sense_key":"answer%2:34:00::","synset_id":"answer.v.09"},{"index":10,"sense_key":"answer%2:31:05::","synset_id":"answer.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"answer%2:32:00::","call_ids":["senseval3.d000.s114.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"answer%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002962,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0736330000218}
{"item_id":"senseval3.d000.s114.t004","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s114.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022385,"input_uncached_usd":0.00003825,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.729519375017844}
{"item_id":"senseval3.d000.s114.t005","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d000.s114.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031755,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.007845957996324}
{"item_id":"senseval3.d000.s114.t006","gold_sense_keys":["skepticism%1:09:00::"],"candidates":[{"index":1,"sense_key":"skepticism%1:09:00::","synset_id":"incredulity.n.01"},{"index":2,"sense_key":"skepticism%1:09:01::","synset_id":"agnosticism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"skepticism%1:09:00::","call_ids":["senseval3.d000.s114.t006__v1__a1","senseval3.d000.s114.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"skepticism%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006408499999999999,"input_uncached_usd":0.00003924999999999999,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.805701332981698}
{"item_id":"senseval3.d000.s114.t007","gold_sense_keys":["show%2:39:01::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"show%2:32:02::","call_ids":["senseval3.d000.s114.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"show%2:32:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":866,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025489999999999996,"input_uncached_usd":0.000043299999999999995,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5410290409927256}
{"item_id":"senseval3.d000.s114.t008","gold_sense_keys":["face%1:07:03::","face%1:08:00::"],"candidates":[{"index":1,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":2,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":3,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":4,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":5,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":6,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":7,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":8,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":9,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":10,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":11,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":12,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":13,"sense_key":"face%1:06:00::","synset_id":"face.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"face%1:07:03::","call_ids":["senseval3.d000.s114.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"face%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017509999999999997,"input_uncached_usd":0.0000407,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.164400250010658}
{"item_id":"senseval3.d000.s114.t009","gold_sense_keys":["voice%1:07:00::"],"candidates":[{"index":1,"sense_key":"voice%1:07:00::","synset_id":"voice.n.01"},{"index":2,"sense_key":"voice%1:10:00::","synset_id":"voice.n.02"},{"index":3,"sense_key":"voice%1:11:00::","synset_id":"voice.n.03"},{"index":4,"sense_key":"voice%1:10:01::","synset_id":"articulation.n.03"},{"index":5,"sense_key":"voice%1:04:00::","synset_id":"voice.n.05"},{"index":6,"sense_key":"voice%1:10:03::","synset_id":"voice.n.06"},{"index":7,"sense_key":"voice%1:18:01::","synset_id":"voice.n.07"},{"index":8,"sense_key":"voice%1:18:00::","synset_id":"spokesperson.n.01"},{"index":9,"sense_key":"voice%1:07:01::","synset_id":"voice.n.09"},{"index":10,"sense_key":"voice%1:24:00::","synset_id":"voice.n.10"},{"index":11,"sense_key":"voice%1:10:02::","synset_id":"part.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"voice%1:07:00::","call_ids":["senseval3.d000.s114.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"voice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.00091555,"input_uncached_usd":0.00003875,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.371079459029716}
{"item_id":"senseval3.d000.s115.t001","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"become%2:30:00::","call_ids":["senseval3.d000.s115.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"become%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036214999999999994,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.632344458019361}
{"item_id":"senseval3.d000.s115.t002","gold_sense_keys":["sad%3:00:00::"],"candidates":[{"index":1,"sense_key":"sad%3:00:00::","synset_id":"sad.a.01"},{"index":2,"sense_key":"sad%5:00:00:sorrowful:00","synset_id":"sad.s.01"},{"index":3,"sense_key":"sad%5:00:00:bad:00","synset_id":"deplorable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sad%3:00:00::","call_ids":["senseval3.d000.s115.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sad%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003366,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8673823330318555}
{"item_id":"senseval3.d000.s115.t003","gold_sense_keys":["moody%5:00:00:ill-natured:00"],"candidates":[{"index":1,"sense_key":"moody%5:00:00:ill-natured:00","synset_id":"dark.s.04"},{"index":2,"sense_key":"moody%5:00:00:emotional:00","synset_id":"moody.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"moody%5:00:00:emotional:00","call_ids":["senseval3.d000.s115.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"moody%5:00:00:emotional:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028284999999999996,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.61303108301945}
{"item_id":"senseval3.d000.s116.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["senseval3.d000.s116.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038865,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.572742332995404}
{"item_id":"senseval3.d000.s116.t002","gold_sense_keys":["company%1:26:00::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"company%1:26:00::","call_ids":["senseval3.d000.s116.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"company%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011475,"input_uncached_usd":0.00003155,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2233149579842575}
{"item_id":"senseval3.d000.s116.t003","gold_sense_keys":["feel%2:31:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"feel%2:31:00::","call_ids":["senseval3.d000.s116.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"feel%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":825,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035485,"input_uncached_usd":0.00004125,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.837675624992698}
{"item_id":"senseval3.d000.s116.t004","gold_sense_keys":["buy%2:40:00::"],"candidates":[{"index":1,"sense_key":"buy%2:40:00::","synset_id":"buy.v.01"},{"index":2,"sense_key":"buy%2:40:02::","synset_id":"bribe.v.01"},{"index":3,"sense_key":"buy%2:42:00::","synset_id":"buy.v.03"},{"index":4,"sense_key":"buy%2:40:03::","synset_id":"buy.v.04"},{"index":5,"sense_key":"buy%2:31:09::","synset_id":"buy.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"buy%2:40:00::","call_ids":["senseval3.d000.s116.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"buy%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033775,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.167986375046894}
{"item_id":"senseval3.d000.s116.t005","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s116.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026155,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.520567458006553}
{"item_id":"senseval3.d000.s116.t006","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"prevent%2:41:01::","call_ids":["senseval3.d000.s116.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"prevent%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046154999999999997,"input_uncached_usd":0.00001995,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.732910750026349}
{"item_id":"senseval3.d000.s116.t007","gold_sense_keys":["possible%3:00:04::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"possible%3:00:04::","call_ids":["senseval3.d000.s116.t007__v1__a1","senseval3.d000.s116.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"possible%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":3296,"reasoning_output_tokens":3264},"cost":{"currency":"USD","total_usd":0.00135585,"input_uncached_usd":0.00003745,"input_cached_usd":0.0,"output_usd":0.0013184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":24.271762042015325}
{"item_id":"senseval3.d000.s116.t008","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::","trouble%1:11:01::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d000.s116.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046744999999999995,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.557583499990869}
{"item_id":"senseval3.d000.s117.t000","gold_sense_keys":["trouble%1:04:02::","trouble%1:09:00::","trouble%1:11:00::","trouble%1:11:01::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d000.s117.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028859999999999997,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9967497919569723}
{"item_id":"senseval3.d000.s118.t000","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s118.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003341,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.465763165964745}
{"item_id":"senseval3.d000.s118.t002","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s118.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023574999999999998,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.070978083007503}
{"item_id":"senseval3.d000.s118.t003","gold_sense_keys":["upset%2:37:00::","upset%2:37:01::"],"candidates":[{"index":1,"sense_key":"upset%2:30:00::","synset_id":"upset.v.01"},{"index":2,"sense_key":"upset%2:37:01::","synset_id":"upset.v.02"},{"index":3,"sense_key":"upset%2:37:00::","synset_id":"disturb.v.01"},{"index":4,"sense_key":"upset%2:38:00::","synset_id":"overturn.v.02"},{"index":5,"sense_key":"upset%2:35:00::","synset_id":"swage.v.01"},{"index":6,"sense_key":"upset%2:33:00::","synset_id":"upset.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"upset%2:37:00::","call_ids":["senseval3.d000.s118.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"upset%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028849999999999997,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.956736667023506}
{"item_id":"senseval3.d000.s118.t005","gold_sense_keys":["appreciation%1:10:00::"],"candidates":[{"index":1,"sense_key":"appreciation%1:09:00::","synset_id":"appreciation.n.01"},{"index":2,"sense_key":"appreciation%1:09:01::","synset_id":"taste.n.03"},{"index":3,"sense_key":"appreciation%1:10:00::","synset_id":"appreciation.n.03"},{"index":4,"sense_key":"appreciation%1:04:00::","synset_id":"admiration.n.03"},{"index":5,"sense_key":"appreciation%1:04:01::","synset_id":"appreciation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appreciation%1:10:00::","call_ids":["senseval3.d000.s118.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appreciation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002876,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.144963333033957}
{"item_id":"senseval3.d000.s119.t000","gold_sense_keys":["break%2:30:03::","break%2:42:05::"],"candidates":[{"index":1,"sense_key":"break%2:30:03::","synset_id":"interrupt.v.04"},{"index":2,"sense_key":"break%2:30:00::","synset_id":"break.v.02"},{"index":3,"sense_key":"break%2:30:06::","synset_id":"break.v.03"},{"index":4,"sense_key":"break%2:35:01::","synset_id":"break.v.04"},{"index":5,"sense_key":"break%2:30:10::","synset_id":"break.v.05"},{"index":6,"sense_key":"break%2:41:00::","synset_id":"transgress.v.01"},{"index":7,"sense_key":"break%2:38:02::","synset_id":"break.v.07"},{"index":8,"sense_key":"break%2:38:08::","synset_id":"break.v.08"},{"index":9,"sense_key":"break%2:37:00::","synset_id":"break.v.09"},{"index":10,"sense_key":"break%2:30:02::","synset_id":"break.v.10"},{"index":11,"sense_key":"break%2:41:11::","synset_id":"break_in.v.01"},{"index":12,"sense_key":"break%2:30:11::","synset_id":"break_in.v.06"},{"index":13,"sense_key":"break%2:42:00::","synset_id":"violate.v.01"},{"index":14,"sense_key":"break%2:33:00::","synset_id":"better.v.01"},{"index":15,"sense_key":"break%2:32:00::","synset_id":"unwrap.v.02"},{"index":16,"sense_key":"break%2:30:04::","synset_id":"break.v.16"},{"index":17,"sense_key":"break%2:30:05::","synset_id":"fail.v.04"},{"index":18,"sense_key":"break%2:41:01::","synset_id":"break.v.18"},{"index":19,"sense_key":"break%2:38:10::","synset_id":"break.v.19"},{"index":20,"sense_key":"break%2:38:01::","synset_id":"break.v.20"},{"index":21,"sense_key":"break%2:30:09::","synset_id":"dampen.v.07"},{"index":22,"sense_key":"break%2:30:12::","synset_id":"break.v.22"},{"index":23,"sense_key":"break%2:42:15::","synset_id":"break.v.23"},{"index":24,"sense_key":"break%2:42:05::","synset_id":"break.v.24"},{"index":25,"sense_key":"break%2:42:04::","synset_id":"break.v.25"},{"index":26,"sense_key":"break%2:42:03::","synset_id":"break.v.26"},{"index":27,"sense_key":"break%2:42:14::","synset_id":"break.v.27"},{"index":28,"sense_key":"break%2:41:15::","synset_id":"break.v.28"},{"index":29,"sense_key":"break%2:41:09::","synset_id":"break.v.29"},{"index":30,"sense_key":"break%2:41:10::","synset_id":"break.v.30"},{"index":31,"sense_key":"break%2:41:13::","synset_id":"separate.v.08"},{"index":32,"sense_key":"break%2:41:06::","synset_id":"demote.v.01"},{"index":33,"sense_key":"break%2:40:07::","synset_id":"bankrupt.v.01"},{"index":34,"sense_key":"break%2:38:15::","synset_id":"break.v.34"},{"index":35,"sense_key":"break%2:38:09::","synset_id":"break.v.35"},{"index":36,"sense_key":"break%2:38:11::","synset_id":"collapse.v.01"},{"index":37,"sense_key":"break%2:38:00::","synset_id":"break_dance.v.01"},{"index":38,"sense_key":"break%2:35:11::","synset_id":"break.v.38"},{"index":39,"sense_key":"break%2:35:10::","synset_id":"break.v.39"},{"index":40,"sense_key":"break%2:35:14::","synset_id":"break.v.40"},{"index":41,"sense_key":"break%2:35:04::","synset_id":"break.v.41"},{"index":42,"sense_key":"break%2:35:00::","synset_id":"break.v.42"},{"index":43,"sense_key":"break%2:35:05::","synset_id":"break.v.43"},{"index":44,"sense_key":"break%2:35:13::","synset_id":"break.v.44"},{"index":45,"sense_key":"break%2:35:09::","synset_id":"break.v.45"},{"index":46,"sense_key":"break%2:32:05::","synset_id":"break.v.46"},{"index":47,"sense_key":"break%2:32:04::","synset_id":"pause.v.02"},{"index":48,"sense_key":"break%2:32:13::","synset_id":"break.v.48"},{"index":49,"sense_key":"break%2:32:07::","synset_id":"break.v.49"},{"index":50,"sense_key":"break%2:31:06::","synset_id":"break.v.50"},{"index":51,"sense_key":"break%2:31:05::","synset_id":"break.v.51"},{"index":52,"sense_key":"break%2:30:14::","synset_id":"break.v.52"},{"index":53,"sense_key":"break%2:30:01::","synset_id":"break.v.53"},{"index":54,"sense_key":"break%2:30:15::","synset_id":"crack.v.01"},{"index":55,"sense_key":"break%2:30:07::","synset_id":"break.v.55"},{"index":56,"sense_key":"break%2:30:08::","synset_id":"break.v.56"},{"index":57,"sense_key":"break%2:29:04::","synset_id":"fracture.v.06"},{"index":58,"sense_key":"break%2:29:11::","synset_id":"break.v.58"},{"index":59,"sense_key":"break%2:29:10::","synset_id":"break.v.59"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":18,"chosen_sense_key":"break%2:41:01::","call_ids":["senseval3.d000.s119.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":18,"predicted_sense_key":"break%2:41:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2668,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0007030000000000001,"input_uncached_usd":0.0001334,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.526741832960397}
{"item_id":"senseval3.d000.s119.t001","gold_sense_keys":["uncomfortable%3:00:01::"],"candidates":[{"index":1,"sense_key":"uncomfortable%3:00:01::","synset_id":"uncomfortable.a.01"},{"index":2,"sense_key":"uncomfortable%3:00:00::","synset_id":"uncomfortable.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uncomfortable%3:00:01::","call_ids":["senseval3.d000.s119.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uncomfortable%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004092,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.303641625039745}
{"item_id":"senseval3.d000.s119.t002","gold_sense_keys":["silence%1:26:00::"],"candidates":[{"index":1,"sense_key":"silence%1:26:00::","synset_id":"silence.n.01"},{"index":2,"sense_key":"silence%1:07:00::","synset_id":"silence.n.02"},{"index":3,"sense_key":"silence%1:07:01::","synset_id":"muteness.n.02"},{"index":4,"sense_key":"silence%1:07:02::","synset_id":"secrecy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"silence%1:07:00::","call_ids":["senseval3.d000.s119.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"silence%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018265,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.581620707991533}
{"item_id":"senseval3.d000.s119.t003","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"begin%2:30:01::","call_ids":["senseval3.d000.s119.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"begin%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025144999999999996,"input_uncached_usd":0.000040249999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.937550583970733}
{"item_id":"senseval3.d000.s120.t000","gold_sense_keys":["drink%1:13:04::"],"candidates":[{"index":1,"sense_key":"drink%1:13:04::","synset_id":"drink.n.01"},{"index":2,"sense_key":"drink%1:04:00::","synset_id":"drink.n.02"},{"index":3,"sense_key":"drink%1:13:00::","synset_id":"beverage.n.01"},{"index":4,"sense_key":"drink%1:17:00::","synset_id":"drink.n.04"},{"index":5,"sense_key":"drink%1:04:01::","synset_id":"swallow.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drink%1:13:04::","call_ids":["senseval3.d000.s120.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drink%1:13:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038974999999999996,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.222006666008383}
{"item_id":"senseval3.d000.s120.t001","gold_sense_keys":["complain%2:32:00::"],"candidates":[{"index":1,"sense_key":"complain%2:32:00::","synset_id":"complain.v.01"},{"index":2,"sense_key":"complain%2:32:01::","synset_id":"complain.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"complain%2:32:00::","call_ids":["senseval3.d000.s120.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"complain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038419999999999996,"input_uncached_usd":0.0000194,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.259152874990832}
{"item_id":"senseval3.d000.s120.t003","gold_sense_keys":["subject%1:09:01::","subject%1:10:00::"],"candidates":[{"index":1,"sense_key":"subject%1:10:00::","synset_id":"subject.n.01"},{"index":2,"sense_key":"subject%1:06:00::","synset_id":"subject.n.02"},{"index":3,"sense_key":"subject%1:09:00::","synset_id":"discipline.n.01"},{"index":4,"sense_key":"subject%1:09:01::","synset_id":"topic.n.02"},{"index":5,"sense_key":"subject%1:10:01::","synset_id":"subject.n.05"},{"index":6,"sense_key":"subject%1:18:01::","synset_id":"subject.n.06"},{"index":7,"sense_key":"subject%1:18:00::","synset_id":"national.n.01"},{"index":8,"sense_key":"subject%1:10:02::","synset_id":"subject.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subject%1:10:00::","call_ids":["senseval3.d000.s120.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subject%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047499999999999994,"input_uncached_usd":0.0000334,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5541647910140455}
{"item_id":"senseval3.d000.s120.t004","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d000.s120.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033995,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.650000708003063}
{"item_id":"senseval3.d000.s120.t005","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d000.s120.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004914,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1077046660357155}
{"item_id":"senseval3.d000.s120.t007","gold_sense_keys":["interest%1:09:00::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"interest%1:07:02::","call_ids":["senseval3.d000.s120.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"interest%1:07:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047219999999999993,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.243021790985949}
{"item_id":"senseval3.d000.s120.t009","gold_sense_keys":["show%2:39:01::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"show%2:39:01::","call_ids":["senseval3.d000.s120.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"show%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004587,"input_uncached_usd":0.0000427,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.399476624967065}
{"item_id":"senseval3.d000.s120.t010","gold_sense_keys":["alert%3:00:00::","alert%5:00:00:aware:00"],"candidates":[{"index":1,"sense_key":"alert%3:00:00::","synset_id":"alert.a.01"},{"index":2,"sense_key":"alert%5:00:00:energetic:00","synset_id":"alert.s.01"},{"index":3,"sense_key":"alert%5:00:00:aware:00","synset_id":"alert.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"alert%5:00:00:aware:00","call_ids":["senseval3.d000.s120.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"alert%5:00:00:aware:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007905,"input_uncached_usd":0.00002145,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0236049169907346}
{"item_id":"senseval3.d000.s120.t011","gold_sense_keys":["face%1:07:03::"],"candidates":[{"index":1,"sense_key":"face%1:08:00::","synset_id":"face.n.01"},{"index":2,"sense_key":"face%1:07:03::","synset_id":"expression.n.01"},{"index":3,"sense_key":"face%1:07:00::","synset_id":"face.n.03"},{"index":4,"sense_key":"face%1:06:02::","synset_id":"face.n.04"},{"index":5,"sense_key":"face%1:18:00::","synset_id":"face.n.05"},{"index":6,"sense_key":"face%1:15:00::","synset_id":"side.n.04"},{"index":7,"sense_key":"face%1:08:01::","synset_id":"face.n.07"},{"index":8,"sense_key":"face%1:06:01::","synset_id":"face.n.08"},{"index":9,"sense_key":"face%1:10:01::","synset_id":"grimace.n.01"},{"index":10,"sense_key":"face%1:10:00::","synset_id":"font.n.01"},{"index":11,"sense_key":"face%1:07:01::","synset_id":"face.n.11"},{"index":12,"sense_key":"face%1:07:02::","synset_id":"boldness.n.02"},{"index":13,"sense_key":"face%1:06:00::","synset_id":"face.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"face%1:07:03::","call_ids":["senseval3.d000.s120.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"face%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020009999999999998,"input_uncached_usd":0.0000401,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.821701042004861}
{"item_id":"senseval3.d000.s120.t012","gold_sense_keys":["bright%5:00:00:happy:00"],"candidates":[{"index":1,"sense_key":"bright%3:00:00::","synset_id":"bright.a.01"},{"index":2,"sense_key":"bright%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":3,"sense_key":"bright%5:00:00:intelligent:00","synset_id":"bright.s.02"},{"index":4,"sense_key":"bright%5:00:00:light:06","synset_id":"bright.s.03"},{"index":5,"sense_key":"bright%5:00:00:polished:00","synset_id":"bright.s.04"},{"index":6,"sense_key":"bright%5:00:00:glorious:00","synset_id":"bright.s.05"},{"index":7,"sense_key":"bright%3:00:02::","synset_id":"undimmed.a.01"},{"index":8,"sense_key":"bright%5:00:00:reverberant:00","synset_id":"bright.s.06"},{"index":9,"sense_key":"bright%5:00:00:happy:00","synset_id":"bright.s.07"},{"index":10,"sense_key":"bright%5:00:00:auspicious:00","synset_id":"bright.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bright%5:00:00:intelligent:00","call_ids":["senseval3.d000.s120.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bright%5:00:00:intelligent:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":2132,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.0008908999999999999,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.0008527999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.304874167020898}
{"item_id":"senseval3.d000.s120.t013","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s120.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020954999999999997,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.471607582992874}
{"item_id":"senseval3.d000.s121.t000","gold_sense_keys":["pause%2:42:00::"],"candidates":[{"index":1,"sense_key":"pause%2:42:00::","synset_id":"hesitate.v.02"},{"index":2,"sense_key":"pause%2:32:01::","synset_id":"pause.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pause%2:42:00::","call_ids":["senseval3.d000.s121.t000__v1__a1","senseval3.d000.s121.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pause%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":865,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038385,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.00034599999999999995,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.788712666020729}
{"item_id":"senseval3.d000.s121.t001","gold_sense_keys":["moisten%2:30:00::"],"candidates":[{"index":1,"sense_key":"moisten%2:30:00::","synset_id":"moisten.v.01"},{"index":2,"sense_key":"moisten%2:35:00::","synset_id":"drizzle.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"moisten%2:30:00::","call_ids":["senseval3.d000.s121.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"moisten%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043384999999999995,"input_uncached_usd":0.00001785,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.362827249977272}
{"item_id":"senseval3.d000.s121.t002","gold_sense_keys":["throat%1:08:00::"],"candidates":[{"index":1,"sense_key":"throat%1:08:00::","synset_id":"throat.n.01"},{"index":2,"sense_key":"throat%1:06:01::","synset_id":"throat.n.02"},{"index":3,"sense_key":"throat%1:06:00::","synset_id":"throat.n.03"},{"index":4,"sense_key":"throat%1:05:00::","synset_id":"throat.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"throat%1:08:00::","call_ids":["senseval3.d000.s121.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"throat%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025814999999999996,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6196685829781927}
{"item_id":"senseval3.d000.s121.t003","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s121.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":342,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048469999999999996,"input_uncached_usd":0.0000171,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.65207858395297}
{"item_id":"senseval3.d000.s121.t004","gold_sense_keys":["break_in%2:32:00::"],"candidates":[{"index":1,"sense_key":"break_in%2:41:00::","synset_id":"break_in.v.01"},{"index":2,"sense_key":"break_in%2:32:00::","synset_id":"chime_in.v.01"},{"index":3,"sense_key":"break_in%2:30:03::","synset_id":"break_in.v.03"},{"index":4,"sense_key":"break_in%2:35:00::","synset_id":"break_in.v.04"},{"index":5,"sense_key":"break_in%2:30:01::","synset_id":"break_in.v.05"},{"index":6,"sense_key":"break_in%2:30:00::","synset_id":"break_in.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"break_in%2:35:00::","call_ids":["senseval3.d000.s121.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"break_in%2:35:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046919999999999997,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.49204224999994}
{"item_id":"senseval3.d000.s122.t000","gold_sense_keys":["pay%2:40:14::"],"candidates":[{"index":1,"sense_key":"pay%2:40:00::","synset_id":"pay.v.01"},{"index":2,"sense_key":"pay%2:32:00::","synset_id":"give.v.05"},{"index":3,"sense_key":"pay%2:40:04::","synset_id":"pay_up.v.01"},{"index":4,"sense_key":"pay%2:40:01::","synset_id":"yield.v.10"},{"index":5,"sense_key":"pay%2:40:02::","synset_id":"pay.v.05"},{"index":6,"sense_key":"pay%2:31:00::","synset_id":"give.v.10"},{"index":7,"sense_key":"pay%2:42:00::","synset_id":"pay.v.07"},{"index":8,"sense_key":"pay%2:41:00::","synset_id":"pay.v.08"},{"index":9,"sense_key":"pay%2:31:03::","synset_id":"pay.v.09"},{"index":10,"sense_key":"pay%2:40:13::","synset_id":"pay.v.10"},{"index":11,"sense_key":"pay%2:40:14::","synset_id":"pay.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pay%2:40:00::","call_ids":["senseval3.d000.s122.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pay%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047654999999999996,"input_uncached_usd":0.000034949999999999996,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.66158837499097}
{"item_id":"senseval3.d000.s122.t001","gold_sense_keys":["bill%1:10:01::"],"candidates":[{"index":1,"sense_key":"bill%1:10:04::","synset_id":"bill.n.01"},{"index":2,"sense_key":"bill%1:10:01::","synset_id":"bill.n.02"},{"index":3,"sense_key":"bill%1:21:00::","synset_id":"bill.n.03"},{"index":4,"sense_key":"bill%1:04:00::","synset_id":"bill.n.04"},{"index":5,"sense_key":"bill%1:10:02::","synset_id":"circular.n.01"},{"index":6,"sense_key":"bill%1:10:00::","synset_id":"poster.n.01"},{"index":7,"sense_key":"bill%1:10:06::","synset_id":"bill.n.07"},{"index":8,"sense_key":"bill%1:06:01::","synset_id":"bill.n.08"},{"index":9,"sense_key":"bill%1:06:00::","synset_id":"bill.n.09"},{"index":10,"sense_key":"bill%1:05:00::","synset_id":"beak.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bill%1:10:01::","call_ids":["senseval3.d000.s122.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bill%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":733,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009390500000000001,"input_uncached_usd":0.000036649999999999996,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.060700875008479}
{"item_id":"senseval3.d000.s123.t001","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s123.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005921500000000001,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.2742604999803}
{"item_id":"senseval3.d000.s123.t002","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"man%1:18:04::","call_ids":["senseval3.d000.s123.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"man%1:18:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045664999999999996,"input_uncached_usd":0.00004065,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.008017582993489}
{"item_id":"senseval3.d000.s123.t004","gold_sense_keys":["say%2:32:01::","say%2:32:13::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s123.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006075500000000001,"input_uncached_usd":0.00003795,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.560367166996002}
{"item_id":"senseval3.d000.s123.t005","gold_sense_keys":["divorce%2:41:00::"],"candidates":[{"index":1,"sense_key":"divorce%2:41:01::","synset_id":"disassociate.v.01"},{"index":2,"sense_key":"divorce%2:41:00::","synset_id":"divorce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"divorce%2:41:00::","call_ids":["senseval3.d000.s123.t005__v1__a1","senseval3.d000.s123.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"divorce%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":1313,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005653499999999999,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.0005252,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.81234795897035}
{"item_id":"senseval3.d000.s125.t000","gold_sense_keys":["bitch%1:18:00::"],"candidates":[{"index":1,"sense_key":"bitch%1:26:00::","synset_id":"bitch.n.01"},{"index":2,"sense_key":"bitch%1:18:00::","synset_id":"cunt.n.01"},{"index":3,"sense_key":"bitch%1:10:00::","synset_id":"gripe.n.01"},{"index":4,"sense_key":"bitch%1:05:00::","synset_id":"bitch.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bitch%1:18:00::","call_ids":["senseval3.d000.s125.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bitch%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004367,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.550073000020348}
{"item_id":"senseval3.d000.s125.t001","gold_sense_keys":["love%2:37:02::"],"candidates":[{"index":1,"sense_key":"love%2:37:00::","synset_id":"love.v.01"},{"index":2,"sense_key":"love%2:37:02::","synset_id":"love.v.02"},{"index":3,"sense_key":"love%2:37:01::","synset_id":"love.v.03"},{"index":4,"sense_key":"love%2:35:00::","synset_id":"sleep_together.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"love%2:37:02::","call_ids":["senseval3.d000.s125.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"love%2:37:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":429,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036064999999999996,"input_uncached_usd":0.00002145,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.838804166007321}
{"item_id":"senseval3.d000.s126.t000","gold_sense_keys":["get%2:40:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"get%2:40:00::","call_ids":["senseval3.d000.s126.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"get%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1860,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0003298,"input_uncached_usd":0.000093,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.939239125000313}
{"item_id":"senseval3.d000.s126.t001","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"},{"index":2,"sense_key":"half%1:28:00::","synset_id":"half.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"half%1:23:00::","call_ids":["senseval3.d000.s126.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035449999999999994,"input_uncached_usd":0.0000153,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.9364468330168165}
{"item_id":"senseval3.d000.s126.t002","gold_sense_keys":["have%2:40:00::","have%2:40:04::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"have%2:40:04::","call_ids":["senseval3.d000.s126.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"have%2:40:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1062,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00013629999999999998,"input_uncached_usd":0.000053099999999999996,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.166480584011879}
{"item_id":"senseval3.d000.s127.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["senseval3.d000.s127.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031945,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.900613958016038}
{"item_id":"senseval3.d000.s128.t001","gold_sense_keys":["business%1:14:00::"],"candidates":[{"index":1,"sense_key":"business%1:14:00::","synset_id":"business.n.01"},{"index":2,"sense_key":"business%1:04:01::","synset_id":"commercial_enterprise.n.02"},{"index":3,"sense_key":"business%1:04:00::","synset_id":"occupation.n.01"},{"index":4,"sense_key":"business%1:09:01::","synset_id":"business.n.04"},{"index":5,"sense_key":"business%1:09:00::","synset_id":"business.n.05"},{"index":6,"sense_key":"business%1:04:02::","synset_id":"business.n.06"},{"index":7,"sense_key":"business%1:14:02::","synset_id":"business.n.07"},{"index":8,"sense_key":"business%1:14:01::","synset_id":"clientele.n.01"},{"index":9,"sense_key":"business%1:04:03::","synset_id":"business.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"business%1:14:00::","call_ids":["senseval3.d000.s128.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"business%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044615,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.056699833017774}
{"item_id":"senseval3.d000.s128.t003","gold_sense_keys":["share%1:21:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":3,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":4,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"share%1:21:00::","call_ids":["senseval3.d000.s128.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"share%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023475,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.350240708037745}
{"item_id":"senseval3.d000.s130.t000","gold_sense_keys":["nod%2:32:00::"],"candidates":[{"index":1,"sense_key":"nod%2:32:01::","synset_id":"nod.v.01"},{"index":2,"sense_key":"nod%2:32:00::","synset_id":"nod.v.02"},{"index":3,"sense_key":"nod%2:29:03::","synset_id":"nod.v.03"},{"index":4,"sense_key":"nod%2:38:00::","synset_id":"nod.v.04"},{"index":5,"sense_key":"nod%2:29:02::","synset_id":"nod.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nod%2:32:01::","call_ids":["senseval3.d000.s130.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nod%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035884999999999997,"input_uncached_usd":0.00001965,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.91035516600823}
{"item_id":"senseval3.d000.s132.t000","gold_sense_keys":["see%2:31:01::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"see%2:31:01::","call_ids":["senseval3.d000.s132.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"see%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1345,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00025285,"input_uncached_usd":0.00006725,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4675808749743737}
{"item_id":"senseval3.d000.s133.t000","gold_sense_keys":["understand%2:31:01::"],"candidates":[{"index":1,"sense_key":"understand%2:31:00::","synset_id":"understand.v.01"},{"index":2,"sense_key":"understand%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"understand%2:31:03::","synset_id":"understand.v.03"},{"index":4,"sense_key":"understand%2:31:02::","synset_id":"understand.v.04"},{"index":5,"sense_key":"understand%2:31:04::","synset_id":"sympathize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"understand%2:31:01::","call_ids":["senseval3.d000.s133.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"understand%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038585,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.187489542004187}
{"item_id":"senseval3.d000.s133.t001","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"state%1:03:00::","call_ids":["senseval3.d000.s133.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"state%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004457,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.6526168750133365}
{"item_id":"senseval3.d000.s134.t000","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d000.s134.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1074,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002653,"input_uncached_usd":0.0000537,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.625725166988559}
{"item_id":"senseval3.d000.s134.t001","gold_sense_keys":["retire%2:41:01::"],"candidates":[{"index":1,"sense_key":"retire%2:41:01::","synset_id":"retire.v.01"},{"index":2,"sense_key":"retire%2:41:02::","synset_id":"retire.v.02"},{"index":3,"sense_key":"retire%2:38:00::","synset_id":"withdraw.v.01"},{"index":4,"sense_key":"retire%2:41:04::","synset_id":"retire.v.04"},{"index":5,"sense_key":"retire%2:41:00::","synset_id":"adjourn.v.02"},{"index":6,"sense_key":"retire%2:41:03::","synset_id":"retire.v.06"},{"index":7,"sense_key":"retire%2:40:00::","synset_id":"retire.v.07"},{"index":8,"sense_key":"retire%2:37:00::","synset_id":"retire.v.08"},{"index":9,"sense_key":"retire%2:35:00::","synset_id":"put_out.v.07"},{"index":10,"sense_key":"retire%2:33:00::","synset_id":"retire.v.10"},{"index":11,"sense_key":"retire%2:29:00::","synset_id":"go_to_bed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"retire%2:41:01::","call_ids":["senseval3.d000.s134.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"retire%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042315,"input_uncached_usd":0.000032749999999999996,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.008481999975629}
{"item_id":"senseval3.d000.s135.t000","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s135.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027685,"input_uncached_usd":0.00004005,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.682927458954509}
{"item_id":"senseval3.d000.s135.t001","gold_sense_keys":["free%5:00:00:unpaid:00"],"candidates":[{"index":1,"sense_key":"free%3:00:00::","synset_id":"free.a.01"},{"index":2,"sense_key":"free%3:00:02::","synset_id":"free.a.02"},{"index":3,"sense_key":"free%5:00:00:unpaid:00","synset_id":"complimentary.s.01"},{"index":4,"sense_key":"free%5:00:02:unoccupied:00","synset_id":"free.s.02"},{"index":5,"sense_key":"free%5:00:00:unfixed:00","synset_id":"detached.s.05"},{"index":6,"sense_key":"free%3:00:01::","synset_id":"free.a.06"},{"index":7,"sense_key":"free%5:00:01:unoccupied:00","synset_id":"spare.s.03"},{"index":8,"sense_key":"free%5:00:00:nonexistent:00","synset_id":"barren.s.03"},{"index":9,"sense_key":"free%5:00:00:inexact:00","synset_id":"free.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"free%5:00:00:unpaid:00","call_ids":["senseval3.d000.s135.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"free%5:00:00:unpaid:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019025,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.562346707971301}
{"item_id":"senseval3.d000.s135.t002","gold_sense_keys":["job%1:04:01::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"job%1:04:01::","call_ids":["senseval3.d000.s135.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"job%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00060725,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.230135832971428}
{"item_id":"senseval3.d000.s136.t000","gold_sense_keys":["tell%2:32:00::","tell%2:32:04::"],"candidates":[{"index":1,"sense_key":"tell%2:32:04::","synset_id":"state.v.01"},{"index":2,"sense_key":"tell%2:32:00::","synset_id":"tell.v.02"},{"index":3,"sense_key":"tell%2:32:01::","synset_id":"tell.v.03"},{"index":4,"sense_key":"tell%2:32:02::","synset_id":"order.v.01"},{"index":5,"sense_key":"tell%2:31:00::","synset_id":"tell.v.05"},{"index":6,"sense_key":"tell%2:32:08::","synset_id":"assure.v.02"},{"index":7,"sense_key":"tell%2:32:03::","synset_id":"tell.v.07"},{"index":8,"sense_key":"tell%2:31:01::","synset_id":"distinguish.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tell%2:32:00::","call_ids":["senseval3.d000.s136.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tell%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039234999999999997,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.937700292037334}
{"item_id":"senseval3.d000.s137.t000","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"feel%2:37:00::","call_ids":["senseval3.d000.s137.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"feel%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005057,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.71485374995973}
{"item_id":"senseval3.d000.s137.t001","gold_sense_keys":["twinge%1:12:00::"],"candidates":[{"index":1,"sense_key":"twinge%1:12:00::","synset_id":"pang.n.01"},{"index":2,"sense_key":"twinge%1:09:00::","synset_id":"twinge.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"twinge%1:12:00::","call_ids":["senseval3.d000.s137.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"twinge%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":291,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022574999999999998,"input_uncached_usd":0.00001455,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.805729084007908}
{"item_id":"senseval3.d000.s137.t002","gold_sense_keys":["annoyance%1:12:00::"],"candidates":[{"index":1,"sense_key":"annoyance%1:26:00::","synset_id":"irritation.n.01"},{"index":2,"sense_key":"annoyance%1:12:00::","synset_id":"annoyance.n.02"},{"index":3,"sense_key":"annoyance%1:18:00::","synset_id":"aggravator.n.01"},{"index":4,"sense_key":"annoyance%1:09:00::","synset_id":"annoyance.n.04"},{"index":5,"sense_key":"annoyance%1:04:00::","synset_id":"annoyance.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"annoyance%1:18:00::","call_ids":["senseval3.d000.s137.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"annoyance%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003358,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.665794417029247}
{"item_id":"senseval3.d000.s137.t003","gold_sense_keys":["hear%2:39:00::"],"candidates":[{"index":1,"sense_key":"hear%2:39:00::","synset_id":"hear.v.01"},{"index":2,"sense_key":"hear%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"hear%2:41:00::","synset_id":"hear.v.03"},{"index":4,"sense_key":"hear%2:39:01::","synset_id":"hear.v.04"},{"index":5,"sense_key":"hear%2:39:05::","synset_id":"listen.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hear%2:39:00::","call_ids":["senseval3.d000.s137.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020815,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.828098875004798}
{"item_id":"senseval3.d000.s137.t004","gold_sense_keys":["familiar%3:00:00::"],"candidates":[{"index":1,"sense_key":"familiar%3:00:00::","synset_id":"familiar.a.01"},{"index":2,"sense_key":"familiar%3:00:02::","synset_id":"familiar.a.02"},{"index":3,"sense_key":"familiar%5:00:00:informed:00","synset_id":"conversant.s.01"},{"index":4,"sense_key":"familiar%5:00:00:close:02","synset_id":"familiar.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"familiar%3:00:00::","call_ids":["senseval3.d000.s137.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"familiar%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003083,"input_uncached_usd":0.0000203,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.24300391698489}
{"item_id":"senseval3.d000.s138.t000","gold_sense_keys":["wild%5:00:00:unrealistic:00"],"candidates":[{"index":1,"sense_key":"wild%3:00:02::","synset_id":"wild.a.01"},{"index":2,"sense_key":"wild%3:00:01::","synset_id":"wild.a.02"},{"index":3,"sense_key":"wild%5:00:00:passionate:00","synset_id":"wild.s.01"},{"index":4,"sense_key":"wild%5:00:01:uncontrolled:00","synset_id":"wild.s.02"},{"index":5,"sense_key":"wild%5:00:00:intense:00","synset_id":"violent.s.02"},{"index":6,"sense_key":"wild%5:00:00:unsupported:02","synset_id":"baseless.s.01"},{"index":7,"sense_key":"wild%5:00:00:insane:00","synset_id":"raving_mad.s.01"},{"index":8,"sense_key":"wild%5:00:00:dangerous:00","synset_id":"hazardous.s.01"},{"index":9,"sense_key":"wild%5:00:00:unrealistic:00","synset_id":"fantastic.s.03"},{"index":10,"sense_key":"wild%5:00:00:inhospitable:00","synset_id":"godforsaken.s.01"},{"index":11,"sense_key":"wild%5:00:00:enthusiastic:00","synset_id":"crazy.s.05"},{"index":12,"sense_key":"wild%5:00:00:noncivilized:00","synset_id":"barbarian.s.01"},{"index":13,"sense_key":"wild%5:00:00:stormy:00","synset_id":"angry.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wild%3:00:02::","call_ids":["senseval3.d000.s138.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wild%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048469999999999996,"input_uncached_usd":0.0000427,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.011984125012532}
{"item_id":"senseval3.d000.s138.t001","gold_sense_keys":["thought%1:09:01::"],"candidates":[{"index":1,"sense_key":"thought%1:09:01::","synset_id":"idea.n.01"},{"index":2,"sense_key":"thought%1:09:00::","synset_id":"thinking.n.01"},{"index":3,"sense_key":"thought%1:09:02::","synset_id":"thought.n.03"},{"index":4,"sense_key":"thought%1:09:03::","synset_id":"opinion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thought%1:09:01::","call_ids":["senseval3.d000.s138.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thought%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056575,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.469785042048898}
{"item_id":"senseval3.d000.s138.t004","gold_sense_keys":["clouded%5:00:00:confused:00"],"candidates":[{"index":1,"sense_key":"clouded%5:00:00:troubled:00","synset_id":"clouded.s.01"},{"index":2,"sense_key":"clouded%5:00:00:cloudy:00","synset_id":"cloud-covered.s.01"},{"index":3,"sense_key":"clouded%5:00:00:confused:00","synset_id":"clouded.s.03"},{"index":4,"sense_key":"clouded%5:00:00:unclear:00","synset_id":"blurred.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"clouded%5:00:00:confused:00","call_ids":["senseval3.d000.s138.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"clouded%5:00:00:confused:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018074999999999998,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1918330000480637}
{"item_id":"senseval3.d000.s138.t005","gold_sense_keys":["brain%1:09:01::"],"candidates":[{"index":1,"sense_key":"brain%1:08:00::","synset_id":"brain.n.01"},{"index":2,"sense_key":"brain%1:09:00::","synset_id":"brain.n.02"},{"index":3,"sense_key":"brain%1:09:01::","synset_id":"mind.n.01"},{"index":4,"sense_key":"brain%1:18:00::","synset_id":"genius.n.01"},{"index":5,"sense_key":"brain%1:13:00::","synset_id":"brain.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"brain%1:09:01::","call_ids":["senseval3.d000.s138.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"brain%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010659999999999999,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2093513750005513}
{"item_id":"senseval3.d000.s139.t000","gold_sense_keys":["suppose%2:32:00::"],"candidates":[{"index":1,"sense_key":"suppose%2:32:00::","synset_id":"suppose.v.01"},{"index":2,"sense_key":"suppose%2:31:00::","synset_id":"think.v.02"},{"index":3,"sense_key":"suppose%2:31:01::","synset_id":"speculate.v.01"},{"index":4,"sense_key":"suppose%2:31:06::","synset_id":"presuppose.v.01"},{"index":5,"sense_key":"suppose%2:31:05::","synset_id":"presuppose.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suppose%2:32:00::","call_ids":["senseval3.d000.s139.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suppose%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049185,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.878607249993365}
{"item_id":"senseval3.d000.s139.t001","gold_sense_keys":["suppose%2:32:00::"],"candidates":[{"index":1,"sense_key":"suppose%2:32:00::","synset_id":"suppose.v.01"},{"index":2,"sense_key":"suppose%2:31:00::","synset_id":"think.v.02"},{"index":3,"sense_key":"suppose%2:31:01::","synset_id":"speculate.v.01"},{"index":4,"sense_key":"suppose%2:31:06::","synset_id":"presuppose.v.01"},{"index":5,"sense_key":"suppose%2:31:05::","synset_id":"presuppose.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suppose%2:31:01::","call_ids":["senseval3.d000.s139.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suppose%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033824999999999995,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.947656499978621}
{"item_id":"senseval3.d000.s139.t002","gold_sense_keys":["guy%1:18:00::"],"candidates":[{"index":1,"sense_key":"guy%1:18:00::","synset_id":"guy.n.01"},{"index":2,"sense_key":"guy%1:06:01::","synset_id":"guy.n.02"},{"index":3,"sense_key":"guy%1:06:00::","synset_id":"guy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guy%1:18:00::","call_ids":["senseval3.d000.s139.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025485,"input_uncached_usd":0.000018049999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.833546042034868}
{"item_id":"senseval3.d000.s139.t003","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s139.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055395,"input_uncached_usd":0.00003555,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.173854999942705}
{"item_id":"senseval3.d000.s140.t001","gold_sense_keys":["professional%3:00:01::"],"candidates":[{"index":1,"sense_key":"professional%3:00:01::","synset_id":"professional.a.01"},{"index":2,"sense_key":"professional%3:01:01::","synset_id":"professional.a.02"},{"index":3,"sense_key":"professional%3:00:02::","synset_id":"professional.a.03"},{"index":4,"sense_key":"professional%3:01:00::","synset_id":"professional.a.04"},{"index":5,"sense_key":"professional%5:00:00:white-collar:00","synset_id":"professional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"professional%3:00:01::","call_ids":["senseval3.d000.s140.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"professional%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002585,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.079488959047012}
{"item_id":"senseval3.d000.s140.t002","gold_sense_keys":["killer%1:18:00::"],"candidates":[{"index":1,"sense_key":"killer%1:18:00::","synset_id":"killer.n.01"},{"index":2,"sense_key":"killer%1:26:00::","synset_id":"cause_of_death.n.01"},{"index":3,"sense_key":"killer%1:09:00::","synset_id":"killer.n.03"},{"index":4,"sense_key":"killer%1:05:00::","synset_id":"killer_whale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"killer%1:18:00::","call_ids":["senseval3.d000.s140.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"killer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005892000000000001,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.486945583950728}
{"item_id":"senseval3.d000.s141.t000","gold_sense_keys":["nut%1:18:02::"],"candidates":[{"index":1,"sense_key":"nut%1:20:00::","synset_id":"nut.n.01"},{"index":2,"sense_key":"nut%1:18:00::","synset_id":"nut.n.02"},{"index":3,"sense_key":"nut%1:06:00::","synset_id":"nut.n.03"},{"index":4,"sense_key":"nut%1:23:00::","synset_id":"en.n.01"},{"index":5,"sense_key":"nut%1:18:02::","synset_id":"crackpot.n.01"},{"index":6,"sense_key":"nut%1:18:01::","synset_id":"addict.n.01"},{"index":7,"sense_key":"nut%1:08:00::","synset_id":"testis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"nut%1:18:02::","call_ids":["senseval3.d000.s141.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"nut%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.000185,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.48116237501381}
{"item_id":"senseval3.d000.s141.t001","gold_sense_keys":["harm%1:11:01::"],"candidates":[{"index":1,"sense_key":"harm%1:26:00::","synset_id":"injury.n.01"},{"index":2,"sense_key":"harm%1:11:01::","synset_id":"damage.n.01"},{"index":3,"sense_key":"harm%1:04:00::","synset_id":"damage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"harm%1:04:00::","call_ids":["senseval3.d000.s141.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"harm%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":339,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056095,"input_uncached_usd":0.00001695,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.915347500005737}
{"item_id":"senseval3.d000.s141.t002","gold_sense_keys":["do%2:36:02::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"do%2:36:02::","call_ids":["senseval3.d000.s141.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"do%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005087,"input_uncached_usd":0.0000415,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.785443458007649}
{"item_id":"senseval3.d000.s142.t001","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d000.s142.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":1809,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007656499999999999,"input_uncached_usd":0.00004205,"input_cached_usd":0.0,"output_usd":0.0007235999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.34348120901268}
{"item_id":"senseval3.d000.s143.t000","gold_sense_keys":["feel%2:29:00::","feel%2:37:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"feel%2:29:00::","call_ids":["senseval3.d000.s143.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"feel%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002758,"input_uncached_usd":0.000039,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.024260542006232}
{"item_id":"senseval3.d000.s143.t001","gold_sense_keys":["cunning%5:00:00:adroit:00","cunning%5:00:00:artful:00"],"candidates":[{"index":1,"sense_key":"cunning%5:00:00:attractive:01","synset_id":"cunning.s.01"},{"index":2,"sense_key":"cunning%5:00:00:artful:00","synset_id":"crafty.s.01"},{"index":3,"sense_key":"cunning%5:00:00:adroit:00","synset_id":"clever.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cunning%5:00:00:artful:00","call_ids":["senseval3.d000.s143.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cunning%5:00:00:artful:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002567,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8643957920139655}
{"item_id":"senseval3.d000.s143.t002","gold_sense_keys":["proud%3:00:00::"],"candidates":[{"index":1,"sense_key":"proud%3:00:00::","synset_id":"proud.a.01"},{"index":2,"sense_key":"proud%5:00:00:impressive:00","synset_id":"gallant.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proud%3:00:00::","call_ids":["senseval3.d000.s143.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proud%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00076565,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.864458583004307}
{"item_id":"senseval3.d000.s143.t003","gold_sense_keys":["play%2:34:13::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"play%2:34:13::","call_ids":["senseval3.d000.s143.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"play%2:34:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1772,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00029979999999999997,"input_uncached_usd":0.0000886,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.453899624990299}
{"item_id":"senseval3.d000.s143.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d000.s143.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033404999999999996,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3433999579865485}
{"item_id":"senseval3.d000.s143.t005","gold_sense_keys":["man%1:18:00::"],"candidates":[{"index":1,"sense_key":"man%1:18:00::","synset_id":"man.n.01"},{"index":2,"sense_key":"man%1:18:03::","synset_id":"serviceman.n.01"},{"index":3,"sense_key":"man%1:18:04::","synset_id":"man.n.03"},{"index":4,"sense_key":"man%1:05:01::","synset_id":"homo.n.02"},{"index":5,"sense_key":"man%1:18:08::","synset_id":"man.n.05"},{"index":6,"sense_key":"man%1:18:06::","synset_id":"man.n.06"},{"index":7,"sense_key":"man%1:18:05::","synset_id":"valet.n.01"},{"index":8,"sense_key":"man%1:18:07::","synset_id":"man.n.08"},{"index":9,"sense_key":"man%1:15:00::","synset_id":"man.n.09"},{"index":10,"sense_key":"man%1:06:00::","synset_id":"man.n.10"},{"index":11,"sense_key":"man%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"man%1:18:00::","call_ids":["senseval3.d000.s143.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"man%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032665,"input_uncached_usd":0.00003865,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.336476499971468}
{"item_id":"senseval3.d000.s143.t006","gold_sense_keys":["soft_spot%1:07:00::"],"candidates":[{"index":1,"sense_key":"soft_spot%1:07:00::","synset_id":"weak_part.n.01"},{"index":2,"sense_key":"soft_spot%1:12:00::","synset_id":"soft_spot.n.02"},{"index":3,"sense_key":"soft_spot%1:08:00::","synset_id":"fontanelle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"soft_spot%1:07:00::","call_ids":["senseval3.d000.s143.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"soft_spot%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":346,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002541,"input_uncached_usd":0.0000173,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.960482833965216}
{"item_id":"senseval3.d000.s144.t000","gold_sense_keys":["offense%1:04:02::"],"candidates":[{"index":1,"sense_key":"offense%1:04:02::","synset_id":"discourtesy.n.03"},{"index":2,"sense_key":"offense%1:12:00::","synset_id":"umbrage.n.01"},{"index":3,"sense_key":"offense%1:04:01::","synset_id":"crime.n.01"},{"index":4,"sense_key":"offense%1:14:00::","synset_id":"offense.n.04"},{"index":5,"sense_key":"offense%1:04:00::","synset_id":"offense.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"offense%1:04:02::","call_ids":["senseval3.d000.s144.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"offense%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028484999999999996,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.691536958969664}
{"item_id":"senseval3.d000.s144.t001","gold_sense_keys":["intend%2:31:00::"],"candidates":[{"index":1,"sense_key":"intend%2:31:00::","synset_id":"intend.v.01"},{"index":2,"sense_key":"intend%2:31:01::","synset_id":"intend.v.02"},{"index":3,"sense_key":"intend%2:32:00::","synset_id":"mean.v.01"},{"index":4,"sense_key":"intend%2:32:02::","synset_id":"mean.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intend%2:31:00::","call_ids":["senseval3.d000.s144.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intend%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.000616,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.495476583018899}
{"item_id":"senseval3.d000.s144.t002","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s144.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":718,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003239,"input_uncached_usd":0.0000359,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.224642459012102}
{"item_id":"senseval3.d000.s146.t001","gold_sense_keys":["suspect%2:32:00::"],"candidates":[{"index":1,"sense_key":"suspect%2:32:00::","synset_id":"suspect.v.01"},{"index":2,"sense_key":"suspect%2:31:00::","synset_id":"distrust.v.01"},{"index":3,"sense_key":"suspect%2:32:02::","synset_id":"suspect.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"suspect%2:32:00::","call_ids":["senseval3.d000.s146.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"suspect%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":1234,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005126499999999999,"input_uncached_usd":0.00001905,"input_cached_usd":0.0,"output_usd":0.0004936,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.047040749981534}
{"item_id":"senseval3.d000.s146.t002","gold_sense_keys":["thing%1:06:01::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"thing%1:06:01::","call_ids":["senseval3.d000.s146.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"thing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":3728,"reasoning_output_tokens":3712},"cost":{"currency":"USD","total_usd":0.0015229999999999998,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.0014912,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":19.03343158395728}
{"item_id":"senseval3.d000.s146.t003","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s146.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":808,"cached_input_tokens":0,"output_tokens":2512,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.0010451999999999998,"input_uncached_usd":0.0000404,"input_cached_usd":0.0,"output_usd":0.0010048,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.217707249976229}
{"item_id":"senseval3.d000.s146.t004","gold_sense_keys":["coincidence%1:11:00::"],"candidates":[{"index":1,"sense_key":"coincidence%1:11:00::","synset_id":"coincidence.n.01"},{"index":2,"sense_key":"coincidence%1:07:02::","synset_id":"coincidence.n.02"},{"index":3,"sense_key":"coincidence%1:07:01::","synset_id":"concurrence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"coincidence%1:11:00::","call_ids":["senseval3.d000.s146.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"coincidence%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.00092115,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.993510624975897}
{"item_id":"senseval3.d000.s147.t000","gold_sense_keys":["know%2:31:00::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"know%2:31:00::","call_ids":["senseval3.d000.s147.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"know%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003255,"input_uncached_usd":0.0000375,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.668678542017005}
{"item_id":"senseval3.d000.s150.t000","gold_sense_keys":["say%2:32:01::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d000.s150.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042535,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.343462500022724}
{"item_id":"senseval3.d000.s150.t001","gold_sense_keys":["manage%2:41:09::"],"candidates":[{"index":1,"sense_key":"manage%2:41:09::","synset_id":"pull_off.v.03"},{"index":2,"sense_key":"manage%2:41:00::","synset_id":"manage.v.02"},{"index":3,"sense_key":"manage%2:41:03::","synset_id":"cope.v.01"},{"index":4,"sense_key":"manage%2:41:01::","synset_id":"oversee.v.01"},{"index":5,"sense_key":"manage%2:41:08::","synset_id":"wangle.v.01"},{"index":6,"sense_key":"manage%2:41:02::","synset_id":"do.v.11"},{"index":7,"sense_key":"manage%2:35:03::","synset_id":"wield.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"manage%2:41:09::","call_ids":["senseval3.d000.s150.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"manage%2:41:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046835,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.0313815830159}
{"item_id":"senseval3.d000.s151.t000","gold_sense_keys":["stranger%1:18:01::"],"candidates":[{"index":1,"sense_key":"stranger%1:18:00::","synset_id":"stranger.n.01"},{"index":2,"sense_key":"stranger%1:18:01::","synset_id":"stranger.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stranger%1:18:01::","call_ids":["senseval3.d000.s151.t000__v1__a1","senseval3.d000.s151.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stranger%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":1825,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007590499999999998,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.00073,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.596975166990887}
{"item_id":"senseval3.d000.s151.t001","gold_sense_keys":["hook%2:32:09::"],"candidates":[{"index":1,"sense_key":"hook%2:35:00::","synset_id":"hook.v.01"},{"index":2,"sense_key":"hook%2:40:02::","synset_id":"overcharge.v.01"},{"index":3,"sense_key":"hook%2:36:00::","synset_id":"crochet.v.02"},{"index":4,"sense_key":"hook%2:35:03::","synset_id":"hook.v.04"},{"index":5,"sense_key":"hook%2:40:01::","synset_id":"hook.v.05"},{"index":6,"sense_key":"hook%2:40:00::","synset_id":"pilfer.v.01"},{"index":7,"sense_key":"hook%2:35:02::","synset_id":"hook.v.07"},{"index":8,"sense_key":"hook%2:35:01::","synset_id":"hook.v.08"},{"index":9,"sense_key":"hook%2:34:00::","synset_id":"addict.v.01"},{"index":10,"sense_key":"hook%2:33:00::","synset_id":"hook.v.10"},{"index":11,"sense_key":"hook%2:32:09::","synset_id":"hook.v.11"},{"index":12,"sense_key":"hook%2:32:00::","synset_id":"hook.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"hook%2:34:00::","call_ids":["senseval3.d000.s151.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"hook%2:34:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004996,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.581817792030051}
{"item_id":"senseval3.d000.s152.t000","gold_sense_keys":["eye%1:08:00::"],"candidates":[{"index":1,"sense_key":"eye%1:08:00::","synset_id":"eye.n.01"},{"index":2,"sense_key":"eye%1:09:00::","synset_id":"eye.n.02"},{"index":3,"sense_key":"eye%1:09:01::","synset_id":"eye.n.03"},{"index":4,"sense_key":"eye%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"eye%1:06:00::","synset_id":"eye.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"eye%1:08:00::","call_ids":["senseval3.d000.s152.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"eye%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025705,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.755790708004497}
{"item_id":"senseval3.d000.s152.t001","gold_sense_keys":["burn%2:43:01::"],"candidates":[{"index":1,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":2,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":3,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":4,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":5,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":6,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":7,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":8,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":9,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":10,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":11,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":12,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":13,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":14,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":15,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"burn%2:37:00::","call_ids":["senseval3.d000.s152.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"burn%2:37:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002247,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5487957500154153}
{"item_id":"senseval3.d000.s153.t000","gold_sense_keys":["mutter%2:32:01::"],"candidates":[{"index":1,"sense_key":"mutter%2:32:01::","synset_id":"mumble.v.01"},{"index":2,"sense_key":"mutter%2:32:00::","synset_id":"murmur.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mutter%2:32:01::","call_ids":["senseval3.d000.s153.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mutter%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":289,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00058445,"input_uncached_usd":0.000014449999999999999,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.263270707975607}
{"item_id":"senseval3.d000.s154.t001","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d000.s154.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019279999999999997,"input_uncached_usd":0.0000328,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9418857079581358}
{"item_id":"senseval3.d000.s155.t001","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["senseval3.d000.s155.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039904999999999997,"input_uncached_usd":0.00003425,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.858973334019538}
{"item_id":"senseval3.d000.s155.t002","gold_sense_keys":["time%1:28:06::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"time%1:28:06::","call_ids":["senseval3.d000.s155.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"time%1:28:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039575,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.245165082975291}
{"item_id":"senseval3.d000.s155.t004","gold_sense_keys":["have%2:40:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d000.s155.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1050,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041729999999999995,"input_uncached_usd":0.000052499999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.383650208008476}
{"item_id":"senseval3.d000.s155.t005","gold_sense_keys":["alibi%1:10:00::"],"candidates":[{"index":1,"sense_key":"alibi%1:10:00::","synset_id":"alibi.n.01"},{"index":2,"sense_key":"alibi%1:10:01::","synset_id":"excuse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alibi%1:10:00::","call_ids":["senseval3.d000.s155.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alibi%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":328,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004324,"input_uncached_usd":0.0000164,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.651337874995079}
{"item_id":"senseval3.d000.s156.t000","gold_sense_keys":["let%2:32:02::","let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":3,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":4,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":5,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":6,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"let%2:32:02::","call_ids":["senseval3.d000.s156.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"let%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023569999999999998,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.596732374979183}
{"item_id":"senseval3.d000.s156.t001","gold_sense_keys":["prove%2:31:00::"],"candidates":[{"index":1,"sense_key":"prove%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"prove%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"prove%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"prove%2:31:02::","synset_id":"prove.v.04"},{"index":5,"sense_key":"prove%2:41:03::","synset_id":"test.v.01"},{"index":6,"sense_key":"prove%2:38:01::","synset_id":"rise.v.15"},{"index":7,"sense_key":"prove%2:38:00::","synset_id":"raise.v.15"},{"index":8,"sense_key":"prove%2:36:00::","synset_id":"prove.v.08"},{"index":9,"sense_key":"prove%2:31:01::","synset_id":"prove.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prove%2:31:00::","call_ids":["senseval3.d000.s156.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prove%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003945,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.679969167045783}
{"item_id":"senseval3.d000.s157.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d000.s157.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":784,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040399999999999995,"input_uncached_usd":0.0000392,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.6538903339533135}
{"item_id":"senseval3.d000.s157.t002","gold_sense_keys":["favor%1:04:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%1:04:00::","call_ids":["senseval3.d000.s157.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048629999999999995,"input_uncached_usd":0.0000191,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.222320416010916}
{"item_id":"senseval3.d000.s158.t000","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wait%2:31:00::","call_ids":["senseval3.d000.s158.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wait%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000308,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.784754584019538}
{"item_id":"senseval3.d000.s159.t001","gold_sense_keys":["liquor%1:13:00::"],"candidates":[{"index":1,"sense_key":"liquor%1:13:00::","synset_id":"liquor.n.01"},{"index":2,"sense_key":"liquor%1:27:00::","synset_id":"liquor.n.02"},{"index":3,"sense_key":"liquor%1:13:01::","synset_id":"liquor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"liquor%1:13:00::","call_ids":["senseval3.d000.s159.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"liquor%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":346,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035649999999999994,"input_uncached_usd":0.0000173,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.037369166035205}
{"item_id":"senseval3.d000.s159.t002","gold_sense_keys":["bottom%1:15:01::"],"candidates":[{"index":1,"sense_key":"bottom%1:15:00::","synset_id":"bottom.n.01"},{"index":2,"sense_key":"bottom%1:15:01::","synset_id":"bottom.n.02"},{"index":3,"sense_key":"bottom%1:08:00::","synset_id":"buttocks.n.01"},{"index":4,"sense_key":"bottom%1:28:00::","synset_id":"bottom.n.04"},{"index":5,"sense_key":"bottom%1:17:00::","synset_id":"bed.n.03"},{"index":6,"sense_key":"bottom%1:17:01::","synset_id":"bottomland.n.01"},{"index":7,"sense_key":"bottom%1:06:00::","synset_id":"bottom.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bottom%1:15:01::","call_ids":["senseval3.d000.s159.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bottom%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021055,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.461706999980379}
{"item_id":"senseval3.d000.s159.t003","gold_sense_keys":["glass%1:06:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"glass%1:06:00::","call_ids":["senseval3.d000.s159.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"glass%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001315,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0222819999908097}
{"item_id":"senseval3.d001.s000.t000","gold_sense_keys":["rout%1:11:00::"],"candidates":[{"index":1,"sense_key":"rout%1:14:00::","synset_id":"mob.n.01"},{"index":2,"sense_key":"rout%1:11:00::","synset_id":"rout.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rout%1:11:00::","call_ids":["senseval3.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rout%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":288,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045599999999999997,"input_uncached_usd":0.0000144,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.171231749991421}
{"item_id":"senseval3.d001.s000.t004","gold_sense_keys":["back%2:41:00::"],"candidates":[{"index":1,"sense_key":"back%2:41:00::","synset_id":"back.v.01"},{"index":2,"sense_key":"back%2:38:00::","synset_id":"back.v.02"},{"index":3,"sense_key":"back%2:41:01::","synset_id":"second.v.01"},{"index":4,"sense_key":"back%2:38:01::","synset_id":"back.v.04"},{"index":5,"sense_key":"back%2:40:00::","synset_id":"back.v.05"},{"index":6,"sense_key":"back%2:42:00::","synset_id":"back.v.06"},{"index":7,"sense_key":"back%2:33:00::","synset_id":"bet_on.v.01"},{"index":8,"sense_key":"back%2:30:03::","synset_id":"back.v.08"},{"index":9,"sense_key":"back%2:30:00::","synset_id":"back.v.09"},{"index":10,"sense_key":"back%2:30:01::","synset_id":"back.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"back%2:41:00::","call_ids":["senseval3.d001.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"back%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005232,"input_uncached_usd":0.0000304,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.29115904198261}
{"item_id":"senseval3.d001.s000.t005","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017879999999999998,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8665097499615513}
{"item_id":"senseval3.d001.s000.t006","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s000.t006__v1__a1","senseval3.d001.s000.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00063455,"input_uncached_usd":0.00003295,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.880294499977026}
{"item_id":"senseval3.d001.s000.t007","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":307,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038015,"input_uncached_usd":0.00001535,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.316517667029984}
{"item_id":"senseval3.d001.s000.t008","gold_sense_keys":["reminder%1:09:00::"],"candidates":[{"index":1,"sense_key":"reminder%1:10:00::","synset_id":"reminder.n.01"},{"index":2,"sense_key":"reminder%1:09:00::","synset_id":"reminder.n.02"},{"index":3,"sense_key":"reminder%1:18:00::","synset_id":"admonisher.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reminder%1:10:00::","call_ids":["senseval3.d001.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reminder%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00027915,"input_uncached_usd":0.00001675,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.594568541971967}
{"item_id":"senseval3.d001.s000.t009","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"},{"index":4,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s000.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018275,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.030841708008666}
{"item_id":"senseval3.d001.s000.t010","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"level%1:09:00::","call_ids":["senseval3.d001.s000.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"level%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00070195,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.799054875038564}
{"item_id":"senseval3.d001.s000.t011","gold_sense_keys":["political%3:00:00::"],"candidates":[{"index":1,"sense_key":"political%3:00:00::","synset_id":"political.a.01"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:01:02::","synset_id":"political.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"political%3:00:00::","call_ids":["senseval3.d001.s000.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"political%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017779999999999998,"input_uncached_usd":0.0000178,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2751900000148453}
{"item_id":"senseval3.d001.s000.t013","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s000.t013__v1__a1","senseval3.d001.s000.t013__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040584999999999997,"input_uncached_usd":0.00003465,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.802124000969343}
{"item_id":"senseval3.d001.s001.t000","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":321,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005856500000000001,"input_uncached_usd":0.00001605,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.430706957995426}
{"item_id":"senseval3.d001.s001.t001","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025679999999999995,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.168614791997243}
{"item_id":"senseval3.d001.s001.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["senseval3.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028484999999999996,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.518682250054553}
{"item_id":"senseval3.d001.s001.t004","gold_sense_keys":["nation%1:14:00::"],"candidates":[{"index":1,"sense_key":"nation%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"nation%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"nation%1:18:00::","synset_id":"nation.n.03"},{"index":4,"sense_key":"nation%1:14:02::","synset_id":"nation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nation%1:14:00::","call_ids":["senseval3.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025905,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.491333791986108}
{"item_id":"senseval3.d001.s001.t007","gold_sense_keys":["choose%2:31:00::"],"candidates":[{"index":1,"sense_key":"choose%2:31:00::","synset_id":"choose.v.01"},{"index":2,"sense_key":"choose%2:31:02::","synset_id":"choose.v.02"},{"index":3,"sense_key":"choose%2:31:01::","synset_id":"choose.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"choose%2:31:02::","call_ids":["senseval3.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"choose%2:31:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003086,"input_uncached_usd":0.0000206,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.843923666048795}
{"item_id":"senseval3.d001.s001.t009","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002339,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.149731166020501}
{"item_id":"senseval3.d001.s001.t010","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s001.t010__v1__a1","senseval3.d001.s001.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048254999999999994,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.000448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.641498290991876}
{"item_id":"senseval3.d001.s002.t000","gold_sense_keys":["percentage%1:24:00::"],"candidates":[{"index":1,"sense_key":"percentage%1:24:00::","synset_id":"percentage.n.01"},{"index":2,"sense_key":"percentage%1:21:00::","synset_id":"share.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"percentage%1:24:00::","call_ids":["senseval3.d001.s002.t000__v1__a1","senseval3.d001.s002.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"percentage%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":800,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035864999999999996,"input_uncached_usd":0.000038650000000000004,"input_cached_usd":0.0,"output_usd":0.00031999999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.9401242909953}
{"item_id":"senseval3.d001.s002.t001","gold_sense_keys":["equal%2:42:00::"],"candidates":[{"index":1,"sense_key":"equal%2:42:00::","synset_id":"equal.v.01"},{"index":2,"sense_key":"equal%2:42:01::","synset_id":"equal.v.02"},{"index":3,"sense_key":"equal%2:30:00::","synset_id":"equal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"equal%2:42:00::","call_ids":["senseval3.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"equal%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015575,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.594673875020817}
{"item_id":"senseval3.d001.s003.t000","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d001.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045775,"input_uncached_usd":0.00004175,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.451306167000439}
{"item_id":"senseval3.d001.s003.t002","gold_sense_keys":["tend%2:42:01::"],"candidates":[{"index":1,"sense_key":"tend%2:42:01::","synset_id":"tend.v.01"},{"index":2,"sense_key":"tend%2:41:00::","synset_id":"tend.v.02"},{"index":3,"sense_key":"tend%2:41:01::","synset_id":"tend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"tend%2:42:01::","call_ids":["senseval3.d001.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"tend%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018179999999999997,"input_uncached_usd":0.000021799999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4039012499852106}
{"item_id":"senseval3.d001.s003.t003","gold_sense_keys":["favor%2:41:00::"],"candidates":[{"index":1,"sense_key":"favor%2:41:00::","synset_id":"prefer.v.03"},{"index":2,"sense_key":"favor%2:31:00::","synset_id":"favor.v.02"},{"index":3,"sense_key":"favor%2:41:01::","synset_id":"favor.v.03"},{"index":4,"sense_key":"favor%2:41:03::","synset_id":"privilege.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"favor%2:41:00::","call_ids":["senseval3.d001.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"favor%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031015000000000003,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.205868833989371}
{"item_id":"senseval3.d001.s003.t004","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023199999999999997,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9803682919591665}
{"item_id":"senseval3.d001.s003.t005","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054555,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.831051458022557}
{"item_id":"senseval3.d001.s003.t006","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039285,"input_uncached_usd":0.000028049999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.96818579104729}
{"item_id":"senseval3.d001.s004.t000","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s004.t000__v1__a1","senseval3.d001.s004.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":851,"cached_input_tokens":0,"output_tokens":2464,"reasoning_output_tokens":2432},"cost":{"currency":"USD","total_usd":0.00102815,"input_uncached_usd":0.000042550000000000004,"input_cached_usd":0.0,"output_usd":0.0009856,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.617539917002432}
{"item_id":"senseval3.d001.s004.t001","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041445,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.856696125003509}
{"item_id":"senseval3.d001.s004.t002","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s004.t002__v1__a1","senseval3.d001.s004.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":1952,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008250500000000001,"input_uncached_usd":0.00004425,"input_cached_usd":0.0,"output_usd":0.0007807999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.178409791958984}
{"item_id":"senseval3.d001.s004.t004","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004105,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.006056874990463}
{"item_id":"senseval3.d001.s004.t007","gold_sense_keys":["percentage%1:24:00::"],"candidates":[{"index":1,"sense_key":"percentage%1:24:00::","synset_id":"percentage.n.01"},{"index":2,"sense_key":"percentage%1:21:00::","synset_id":"share.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"percentage%1:24:00::","call_ids":["senseval3.d001.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"percentage%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000258,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.441579749982338}
{"item_id":"senseval3.d001.s004.t008","gold_sense_keys":["major%3:00:01::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval3.d001.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":2961,"reasoning_output_tokens":2944},"cost":{"currency":"USD","total_usd":0.00121425,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0011844,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":21.53330554201966}
{"item_id":"senseval3.d001.s004.t009","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059635,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.696411957964301}
{"item_id":"senseval3.d001.s004.t010","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":2,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":3,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":4,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"popular%5:00:01:common:01","call_ids":["senseval3.d001.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"popular%5:00:01:common:01","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059535,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.453377833007835}
{"item_id":"senseval3.d001.s004.t011","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":2193,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009055999999999999,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0008772,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.949932374991477}
{"item_id":"senseval3.d001.s004.t012","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006495,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.116125166008715}
{"item_id":"senseval3.d001.s004.t014","gold_sense_keys":["seat%1:07:00::"],"candidates":[{"index":1,"sense_key":"seat%1:15:01::","synset_id":"seat.n.01"},{"index":2,"sense_key":"seat%1:08:00::","synset_id":"buttocks.n.01"},{"index":3,"sense_key":"seat%1:06:01::","synset_id":"seat.n.03"},{"index":4,"sense_key":"seat%1:06:00::","synset_id":"seat.n.04"},{"index":5,"sense_key":"seat%1:15:00::","synset_id":"seat.n.05"},{"index":6,"sense_key":"seat%1:15:02::","synset_id":"seat.n.06"},{"index":7,"sense_key":"seat%1:07:00::","synset_id":"seat.n.07"},{"index":8,"sense_key":"seat%1:06:03::","synset_id":"seat.n.08"},{"index":9,"sense_key":"seat%1:06:02::","synset_id":"seat.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"seat%1:07:00::","call_ids":["senseval3.d001.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"seat%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032415,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.884500874963123}
{"item_id":"senseval3.d001.s004.t015","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":2,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":3,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":4,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s004.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026254999999999996,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.431309875042643}
{"item_id":"senseval3.d001.s004.t016","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005980000000000001,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.507312375004403}
{"item_id":"senseval3.d001.s005.t001","gold_sense_keys":["opposite%1:24:03::"],"candidates":[{"index":1,"sense_key":"opposite%1:10:00::","synset_id":"antonym.n.01"},{"index":2,"sense_key":"opposite%1:24:03::","synset_id":"reverse.n.01"},{"index":3,"sense_key":"opposite%1:18:00::","synset_id":"opposition.n.04"},{"index":4,"sense_key":"opposite%1:24:04::","synset_id":"inverse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"opposite%1:24:03::","call_ids":["senseval3.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"opposite%1:24:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004168,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.782212041027378}
{"item_id":"senseval3.d001.s006.t000","gold_sense_keys":["account_for%2:42:00::"],"candidates":[{"index":1,"sense_key":"account_for%2:42:00::","synset_id":"account_for.v.01"},{"index":2,"sense_key":"account_for%2:32:00::","synset_id":"account_for.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"account_for%2:42:00::","call_ids":["senseval3.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"account_for%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004607999999999999,"input_uncached_usd":0.0000192,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.714751875028014}
{"item_id":"senseval3.d001.s006.t001","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["senseval3.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054465,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.274225124972872}
{"item_id":"senseval3.d001.s006.t002","gold_sense_keys":["recent%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"recent%5:00:00:new:00","synset_id":"recent.s.01"},{"index":2,"sense_key":"recent%5:00:00:past:00","synset_id":"late.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"recent%5:00:00:past:00","call_ids":["senseval3.d001.s006.t002__v1__a1","senseval3.d001.s006.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"recent%5:00:00:past:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00069425,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.0006527999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.85364995803684}
{"item_id":"senseval3.d001.s006.t003","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["senseval3.d001.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028215,"input_uncached_usd":0.00001975,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.96643045800738}
{"item_id":"senseval3.d001.s007.t000","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval3.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005737999999999999,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.647838166973088}
{"item_id":"senseval3.d001.s007.t004","gold_sense_keys":["partial%5:00:00:incomplete:00"],"candidates":[{"index":1,"sense_key":"partial%5:00:00:incomplete:00","synset_id":"partial.s.01"},{"index":2,"sense_key":"partial%3:00:01::","synset_id":"partial.a.02"},{"index":3,"sense_key":"partial%5:00:00:inclined:02","synset_id":"fond.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"partial%5:00:00:incomplete:00","call_ids":["senseval3.d001.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"partial%5:00:00:incomplete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017989999999999998,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6304840419907123}
{"item_id":"senseval3.d001.s007.t007","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["senseval3.d001.s007.t007__v1__a1","senseval3.d001.s007.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":2144,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00089705,"input_uncached_usd":0.00003945,"input_cached_usd":0.0,"output_usd":0.0008575999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.047209249052685}
{"item_id":"senseval3.d001.s007.t008","gold_sense_keys":["american%1:18:00::"],"candidates":[{"index":1,"sense_key":"american%1:18:00::","synset_id":"american.n.01"},{"index":2,"sense_key":"american%1:10:00::","synset_id":"american_english.n.01"},{"index":3,"sense_key":"american%1:18:01::","synset_id":"american.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%1:18:00::","call_ids":["senseval3.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":375,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028115,"input_uncached_usd":0.00001875,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.513393333996646}
{"item_id":"senseval3.d001.s008.t002","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007837499999999999,"input_uncached_usd":0.000034949999999999996,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.108731458021794}
{"item_id":"senseval3.d001.s009.t002","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":3,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":4,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":5,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":6,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034455,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.672043500002474}
{"item_id":"senseval3.d001.s009.t004","gold_sense_keys":["political%3:00:00::","political%3:01:02::"],"candidates":[{"index":1,"sense_key":"political%3:00:00::","synset_id":"political.a.01"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:01:02::","synset_id":"political.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"political%3:00:00::","call_ids":["senseval3.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"political%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015999999999999999,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8309639579965733}
{"item_id":"senseval3.d001.s009.t005","gold_sense_keys":["agent%1:18:02::"],"candidates":[{"index":1,"sense_key":"agent%1:17:00::","synset_id":"agent.n.01"},{"index":2,"sense_key":"agent%1:18:02::","synset_id":"agent.n.02"},{"index":3,"sense_key":"agent%1:27:00::","synset_id":"agent.n.03"},{"index":4,"sense_key":"agent%1:18:01::","synset_id":"agent.n.04"},{"index":5,"sense_key":"agent%1:18:03::","synset_id":"agent.n.05"},{"index":6,"sense_key":"agent%1:10:00::","synset_id":"agentive_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"agent%1:18:02::","call_ids":["senseval3.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"agent%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039464999999999997,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.319916125037707}
{"item_id":"senseval3.d001.s009.t006","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s009.t006__v1__a1","senseval3.d001.s009.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0006045499999999999,"input_uncached_usd":0.00005415,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.604355540999677}
{"item_id":"senseval3.d001.s009.t007","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":3,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":4,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":5,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":6,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037014999999999997,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.610232250008266}
{"item_id":"senseval3.d001.s009.t009","gold_sense_keys":["minimalist%5:00:00:moderate:00"],"candidates":[{"index":1,"sense_key":"minimalist%3:01:00::","synset_id":"minimalist.a.01"},{"index":2,"sense_key":"minimalist%5:00:00:moderate:00","synset_id":"minimalist.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minimalist%5:00:00:moderate:00","call_ids":["senseval3.d001.s009.t009__v1__a1","senseval3.d001.s009.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minimalist%5:00:00:moderate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":1824,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00077725,"input_uncached_usd":0.00004764999999999999,"input_cached_usd":0.0,"output_usd":0.0007296,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.372248167055659}
{"item_id":"senseval3.d001.s009.t010","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:00::","call_ids":["senseval3.d001.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004297,"input_uncached_usd":0.0000393,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.3191438330104575}
{"item_id":"senseval3.d001.s009.t011","gold_sense_keys":["virtue%1:07:03::"],"candidates":[{"index":1,"sense_key":"virtue%1:07:01::","synset_id":"virtue.n.01"},{"index":2,"sense_key":"virtue%1:07:03::","synset_id":"merit.n.01"},{"index":3,"sense_key":"virtue%1:07:02::","synset_id":"virtue.n.03"},{"index":4,"sense_key":"virtue%1:07:04::","synset_id":"virtue.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"virtue%1:07:03::","call_ids":["senseval3.d001.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"virtue%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049345,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.964168875012547}
{"item_id":"senseval3.d001.s009.t012","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":2,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"},{"index":3,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":4,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"private%3:00:00::","call_ids":["senseval3.d001.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"private%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028964999999999997,"input_uncached_usd":0.000027249999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.300694750039838}
{"item_id":"senseval3.d001.s009.t013","gold_sense_keys":["market%1:04:00::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"market%1:04:00::","call_ids":["senseval3.d001.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"market%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019055,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2843850839999504}
{"item_id":"senseval3.d001.s009.t014","gold_sense_keys":["vice%1:07:00::"],"candidates":[{"index":1,"sense_key":"vice%1:07:00::","synset_id":"frailty.n.02"},{"index":2,"sense_key":"vice%1:04:00::","synset_id":"vice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vice%1:07:00::","call_ids":["senseval3.d001.s009.t014__v1__a1","senseval3.d001.s009.t014__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vice%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":955,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00070055,"input_uncached_usd":0.000047749999999999995,"input_cached_usd":0.0,"output_usd":0.0006527999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.314993917068932}
{"item_id":"senseval3.d001.s009.t015","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["senseval3.d001.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005675,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.164128333039116}
{"item_id":"senseval3.d001.s009.t016","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"action%1:04:04::","call_ids":["senseval3.d001.s009.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"action%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003561,"input_uncached_usd":0.000042499999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.436247709032614}
{"item_id":"senseval3.d001.s009.t017","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s009.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002638,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.266279583971482}
{"item_id":"senseval3.d001.s009.t018","gold_sense_keys":["brand%1:09:00::"],"candidates":[{"index":1,"sense_key":"brand%1:10:00::","synset_id":"trade_name.n.01"},{"index":2,"sense_key":"brand%1:09:00::","synset_id":"brand.n.02"},{"index":3,"sense_key":"brand%1:10:02::","synset_id":"brand.n.03"},{"index":4,"sense_key":"brand%1:27:00::","synset_id":"brand.n.04"},{"index":5,"sense_key":"brand%1:10:01::","synset_id":"mark.n.06"},{"index":6,"sense_key":"brand%1:06:00::","synset_id":"sword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"brand%1:09:00::","call_ids":["senseval3.d001.s009.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"brand%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031895000000000003,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3658202909864485}
{"item_id":"senseval3.d001.s009.t022","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["senseval3.d001.s009.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006187,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.973861667036545}
{"item_id":"senseval3.d001.s009.t023","gold_sense_keys":["intervention%1:04:00::"],"candidates":[{"index":1,"sense_key":"intervention%1:04:00::","synset_id":"intervention.n.01"},{"index":2,"sense_key":"intervention%1:10:00::","synset_id":"intervention.n.02"},{"index":3,"sense_key":"intervention%1:04:03::","synset_id":"interposition.n.02"},{"index":4,"sense_key":"intervention%1:04:01::","synset_id":"intervention.n.04"},{"index":5,"sense_key":"intervention%1:04:02::","synset_id":"treatment.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"intervention%1:04:03::","call_ids":["senseval3.d001.s009.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"intervention%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":2576,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010618,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.0010304,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.05082891701022}
{"item_id":"senseval3.d001.s009.t024","gold_sense_keys":["remedy%2:30:00::"],"candidates":[{"index":1,"sense_key":"remedy%2:30:00::","synset_id":"rectify.v.04"},{"index":2,"sense_key":"remedy%2:29:00::","synset_id":"remedy.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remedy%2:30:00::","call_ids":["senseval3.d001.s009.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remedy%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041415,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.914160625019576}
{"item_id":"senseval3.d001.s009.t025","gold_sense_keys":["excess%1:07:00::"],"candidates":[{"index":1,"sense_key":"excess%1:07:02::","synset_id":"excess.n.01"},{"index":2,"sense_key":"excess%1:07:00::","synset_id":"excess.n.02"},{"index":3,"sense_key":"excess%1:26:00::","synset_id":"surfeit.n.01"},{"index":4,"sense_key":"excess%1:04:00::","synset_id":"overindulgence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"excess%1:07:00::","call_ids":["senseval3.d001.s009.t025__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"excess%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003922,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.023628292023204}
{"item_id":"senseval3.d001.s009.t026","gold_sense_keys":["attendant%5:00:00:subsequent:00"],"candidates":[{"index":1,"sense_key":"attendant%5:00:00:present:02","synset_id":"attendant.s.01"},{"index":2,"sense_key":"attendant%5:00:00:subsequent:00","synset_id":"attendant.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendant%5:00:00:subsequent:00","call_ids":["senseval3.d001.s009.t026__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendant%5:00:00:subsequent:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046684999999999994,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.359698834014125}
{"item_id":"senseval3.d001.s009.t027","gold_sense_keys":["pursuit%1:09:00::"],"candidates":[{"index":1,"sense_key":"pursuit%1:04:00::","synset_id":"pursuit.n.01"},{"index":2,"sense_key":"pursuit%1:09:00::","synset_id":"pursuit.n.02"},{"index":3,"sense_key":"pursuit%1:04:02::","synset_id":"avocation.n.01"},{"index":4,"sense_key":"pursuit%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pursuit%1:09:00::","call_ids":["senseval3.d001.s009.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pursuit%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041995,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.808384666976053}
{"item_id":"senseval3.d001.s009.t028","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":2,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"},{"index":3,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":4,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"private%3:00:00::","call_ids":["senseval3.d001.s009.t028__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"private%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00067365,"input_uncached_usd":0.000027249999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.581952165986877}
{"item_id":"senseval3.d001.s009.t029","gold_sense_keys":["interest%1:07:01::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"interest%1:21:03::","call_ids":["senseval3.d001.s009.t029__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"interest%1:21:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002473,"input_uncached_usd":0.000036099999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.351595083018765}
{"item_id":"senseval3.d001.s010.t001","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018754999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.2137825840036385}
{"item_id":"senseval3.d001.s010.t002","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1217,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00029765,"input_uncached_usd":0.000060849999999999995,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.271699749981053}
{"item_id":"senseval3.d001.s010.t004","gold_sense_keys":["responsibility%1:04:00::"],"candidates":[{"index":1,"sense_key":"responsibility%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"responsibility%1:26:00::","synset_id":"province.n.02"},{"index":3,"sense_key":"responsibility%1:07:00::","synset_id":"responsibility.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"responsibility%1:07:00::","call_ids":["senseval3.d001.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"responsibility%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029035,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.700160958978813}
{"item_id":"senseval3.d001.s010.t005","gold_sense_keys":["vote%2:41:10::","vote%2:41:11::","vote%2:41:12::","vote%2:41:13::"],"candidates":[{"index":1,"sense_key":"vote%2:41:13::","synset_id":"vote.v.01"},{"index":2,"sense_key":"vote%2:41:10::","synset_id":"vote.v.02"},{"index":3,"sense_key":"vote%2:41:11::","synset_id":"vote.v.03"},{"index":4,"sense_key":"vote%2:41:09::","synset_id":"vote.v.04"},{"index":5,"sense_key":"vote%2:41:12::","synset_id":"vote.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%2:41:13::","call_ids":["senseval3.d001.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%2:41:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008029000000000001,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.452146457973868}
{"item_id":"senseval3.d001.s010.t006","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027204999999999997,"input_uncached_usd":0.000035249999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.678129707986955}
{"item_id":"senseval3.d001.s010.t007","gold_sense_keys":["deal%2:41:09::"],"candidates":[{"index":1,"sense_key":"deal%2:32:08::","synset_id":"cover.v.05"},{"index":2,"sense_key":"deal%2:31:10::","synset_id":"consider.v.03"},{"index":3,"sense_key":"deal%2:41:09::","synset_id":"deal.v.03"},{"index":4,"sense_key":"deal%2:41:00::","synset_id":"cope.v.01"},{"index":5,"sense_key":"deal%2:40:01::","synset_id":"distribute.v.01"},{"index":6,"sense_key":"deal%2:40:00::","synset_id":"deal.v.06"},{"index":7,"sense_key":"deal%2:41:13::","synset_id":"manage.v.02"},{"index":8,"sense_key":"deal%2:41:04::","synset_id":"deal.v.08"},{"index":9,"sense_key":"deal%2:40:02::","synset_id":"deal.v.09"},{"index":10,"sense_key":"deal%2:41:02::","synset_id":"conduct.v.01"},{"index":11,"sense_key":"deal%2:40:07::","synset_id":"share.v.04"},{"index":12,"sense_key":"deal%2:40:06::","synset_id":"deal.v.12"},{"index":13,"sense_key":"deal%2:40:11::","synset_id":"deal.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"deal%2:41:09::","call_ids":["senseval3.d001.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"deal%2:41:09::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046395,"input_uncached_usd":0.00004795,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.682275625003967}
{"item_id":"senseval3.d001.s010.t008","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s010.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024359999999999999,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.63227762497263}
{"item_id":"senseval3.d001.s010.t009","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:00::","call_ids":["senseval3.d001.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020619999999999997,"input_uncached_usd":0.0000462,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.201043040957302}
{"item_id":"senseval3.d001.s010.t010","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"action%1:04:04::","call_ids":["senseval3.d001.s010.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"action%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038109999999999994,"input_uncached_usd":0.000041899999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.510958375001792}
{"item_id":"senseval3.d001.s010.t011","gold_sense_keys":["cast%2:40:13::"],"candidates":[{"index":1,"sense_key":"cast%2:36:02::","synset_id":"project.v.10"},{"index":2,"sense_key":"cast%2:40:13::","synset_id":"cast.v.02"},{"index":3,"sense_key":"cast%2:36:01::","synset_id":"cast.v.03"},{"index":4,"sense_key":"cast%2:35:00::","synset_id":"hurl.v.01"},{"index":5,"sense_key":"cast%2:41:14::","synset_id":"cast.v.05"},{"index":6,"sense_key":"cast%2:38:01::","synset_id":"roll.v.12"},{"index":7,"sense_key":"cast%2:36:00::","synset_id":"cast.v.07"},{"index":8,"sense_key":"cast%2:35:01::","synset_id":"shed.v.01"},{"index":9,"sense_key":"cast%2:33:00::","synset_id":"draw.v.14"},{"index":10,"sense_key":"cast%2:32:00::","synset_id":"frame.v.04"},{"index":11,"sense_key":"cast%2:29:00::","synset_id":"vomit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cast%2:40:13::","call_ids":["senseval3.d001.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cast%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022989999999999998,"input_uncached_usd":0.0000443,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.750494708016049}
{"item_id":"senseval3.d001.s010.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s010.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044675,"input_uncached_usd":0.000030749999999999995,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.820593375014141}
{"item_id":"senseval3.d001.s010.t014","gold_sense_keys":["legislation%1:10:00::"],"candidates":[{"index":1,"sense_key":"legislation%1:10:00::","synset_id":"legislation.n.01"},{"index":2,"sense_key":"legislation%1:04:00::","synset_id":"legislation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"legislation%1:10:00::","call_ids":["senseval3.d001.s010.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"legislation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":978,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041355,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00039119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.832952541008126}
{"item_id":"senseval3.d001.s010.t015","gold_sense_keys":["impose%2:32:00::"],"candidates":[{"index":1,"sense_key":"impose%2:41:00::","synset_id":"enforce.v.02"},{"index":2,"sense_key":"impose%2:32:00::","synset_id":"inflict.v.01"},{"index":3,"sense_key":"impose%2:40:00::","synset_id":"levy.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"impose%2:40:00::","call_ids":["senseval3.d001.s010.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"impose%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.000108,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9525682499515824}
{"item_id":"senseval3.d001.s010.t016","gold_sense_keys":["cost%1:07:01::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"},{"index":3,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cost%1:07:01::","call_ids":["senseval3.d001.s010.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cost%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023784999999999998,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.420739708992187}
{"item_id":"senseval3.d001.s010.t017","gold_sense_keys":["confer%2:40:00::"],"candidates":[{"index":1,"sense_key":"confer%2:32:00::","synset_id":"confer.v.01"},{"index":2,"sense_key":"confer%2:40:00::","synset_id":"confer.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confer%2:40:00::","call_ids":["senseval3.d001.s010.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confer%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038885,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.667491750034969}
{"item_id":"senseval3.d001.s010.t018","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s010.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018385,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2781912919599563}
{"item_id":"senseval3.d001.s010.t019","gold_sense_keys":["population%1:14:00::"],"candidates":[{"index":1,"sense_key":"population%1:14:00::","synset_id":"population.n.01"},{"index":2,"sense_key":"population%1:14:01::","synset_id":"population.n.02"},{"index":3,"sense_key":"population%1:09:00::","synset_id":"population.n.03"},{"index":4,"sense_key":"population%1:23:00::","synset_id":"population.n.04"},{"index":5,"sense_key":"population%1:04:00::","synset_id":"population.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"population%1:14:00::","call_ids":["senseval3.d001.s010.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"population%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002432,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.414139750006143}
{"item_id":"senseval3.d001.s010.t020","gold_sense_keys":["attend_to%2:31:00::"],"candidates":[{"index":1,"sense_key":"attend_to%2:31:00::","synset_id":"attend_to.v.01"},{"index":2,"sense_key":"attend_to%2:41:01::","synset_id":"serve.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend_to%2:31:00::","call_ids":["senseval3.d001.s010.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend_to%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043999999999999996,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5111535000032745}
{"item_id":"senseval3.d001.s010.t021","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s010.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002874,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.546490291017108}
{"item_id":"senseval3.d001.s010.t022","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:00::","call_ids":["senseval3.d001.s010.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":924,"cached_input_tokens":0,"output_tokens":850,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038619999999999995,"input_uncached_usd":0.0000462,"input_cached_usd":0.0,"output_usd":0.00033999999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.613787499954924}
{"item_id":"senseval3.d001.s010.t024","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"service%1:04:00::","call_ids":["senseval3.d001.s010.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"service%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066885,"input_uncached_usd":0.000048049999999999996,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.344563000020571}
{"item_id":"senseval3.d001.s011.t000","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s011.t000__v1__a1","senseval3.d001.s011.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1099,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00065655,"input_uncached_usd":0.000054949999999999994,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.866565249976702}
{"item_id":"senseval3.d001.s011.t002","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021414999999999997,"input_uncached_usd":0.00002855,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.214191959006712}
{"item_id":"senseval3.d001.s011.t003","gold_sense_keys":["belief%1:09:00::"],"candidates":[{"index":1,"sense_key":"belief%1:09:00::","synset_id":"belief.n.01"},{"index":2,"sense_key":"belief%1:09:01::","synset_id":"impression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"belief%1:09:00::","call_ids":["senseval3.d001.s011.t003__v1__a1","senseval3.d001.s011.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"belief%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1003,"cached_input_tokens":0,"output_tokens":2656,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.00111255,"input_uncached_usd":0.00005015,"input_cached_usd":0.0,"output_usd":0.0010623999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":21.235582084045745}
{"item_id":"senseval3.d001.s011.t004","gold_sense_keys":["minimalist%5:00:00:moderate:00"],"candidates":[{"index":1,"sense_key":"minimalist%3:01:00::","synset_id":"minimalist.a.01"},{"index":2,"sense_key":"minimalist%5:00:00:moderate:00","synset_id":"minimalist.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minimalist%5:00:00:moderate:00","call_ids":["senseval3.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minimalist%5:00:00:moderate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046464999999999994,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.48448758298764}
{"item_id":"senseval3.d001.s011.t005","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:00::","call_ids":["senseval3.d001.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":2128,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00089085,"input_uncached_usd":0.00003965,"input_cached_usd":0.0,"output_usd":0.0008512,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.305219917034265}
{"item_id":"senseval3.d001.s011.t006","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"},{"index":2,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s011.t006__v1__a1","senseval3.d001.s011.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1007,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0006007499999999999,"input_uncached_usd":0.000050350000000000004,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.579232082993258}
{"item_id":"senseval3.d001.s011.t007","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":2,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":3,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":4,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":5,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":6,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":7,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":8,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":9,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":10,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":815,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045674999999999996,"input_uncached_usd":0.00004075,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.104076875024475}
{"item_id":"senseval3.d001.s011.t008","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s011.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028845,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.469818958023097}
{"item_id":"senseval3.d001.s011.t009","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046859999999999995,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.67129395797383}
{"item_id":"senseval3.d001.s011.t010","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval3.d001.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029505,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.00493566604564}
{"item_id":"senseval3.d001.s012.t000","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048144999999999997,"input_uncached_usd":0.00003985,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.806750042014755}
{"item_id":"senseval3.d001.s012.t001","gold_sense_keys":["hold%2:42:02::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":15,"chosen_sense_key":"hold%2:42:02::","call_ids":["senseval3.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":15,"predicted_sense_key":"hold%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2064,"cached_input_tokens":0,"output_tokens":2128,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.0009544,"input_uncached_usd":0.0001032,"input_cached_usd":0.0,"output_usd":0.0008512,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.35412766697118}
{"item_id":"senseval3.d001.s012.t003","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014179999999999998,"input_uncached_usd":0.000032999999999999996,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7706395409768447}
{"item_id":"senseval3.d001.s012.t004","gold_sense_keys":["face%2:32:00::"],"candidates":[{"index":1,"sense_key":"face%2:32:00::","synset_id":"confront.v.02"},{"index":2,"sense_key":"face%2:33:00::","synset_id":"confront.v.01"},{"index":3,"sense_key":"face%2:42:00::","synset_id":"front.v.01"},{"index":4,"sense_key":"face%2:42:02::","synset_id":"face.v.04"},{"index":5,"sense_key":"face%2:38:00::","synset_id":"face.v.05"},{"index":6,"sense_key":"face%2:32:03::","synset_id":"confront.v.03"},{"index":7,"sense_key":"face%2:39:04::","synset_id":"face.v.07"},{"index":8,"sense_key":"face%2:35:00::","synset_id":"face.v.08"},{"index":9,"sense_key":"face%2:35:01::","synset_id":"face.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"face%2:32:00::","call_ids":["senseval3.d001.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"face%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030345,"input_uncached_usd":0.000041049999999999995,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.338520041026641}
{"item_id":"senseval3.d001.s012.t007","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1274,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00032609999999999996,"input_uncached_usd":0.0000637,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.788949083012994}
{"item_id":"senseval3.d001.s012.t008","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"},{"index":2,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s012.t008__v1__a1","senseval3.d001.s012.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1061,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00055225,"input_uncached_usd":0.00005305,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.34635387500748}
{"item_id":"senseval3.d001.s012.t010","gold_sense_keys":["lean%2:42:01::"],"candidates":[{"index":1,"sense_key":"lean%2:38:00::","synset_id":"lean.v.01"},{"index":2,"sense_key":"lean%2:35:00::","synset_id":"lean.v.02"},{"index":3,"sense_key":"lean%2:42:01::","synset_id":"tend.v.01"},{"index":4,"sense_key":"lean%2:31:10::","synset_id":"lean.v.04"},{"index":5,"sense_key":"lean%2:38:07::","synset_id":"list.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lean%2:42:01::","call_ids":["senseval3.d001.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lean%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016629999999999998,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5122125840280205}
{"item_id":"senseval3.d001.s012.t011","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013754999999999998,"input_uncached_usd":0.000028749999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.452399624977261}
{"item_id":"senseval3.d001.s013.t000","gold_sense_keys":["put%2:35:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"put%2:35:00::","call_ids":["senseval3.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"put%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006361499999999999,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.778488625015598}
{"item_id":"senseval3.d001.s013.t001","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"office%1:04:02::","call_ids":["senseval3.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"office%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":760,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003264,"input_uncached_usd":0.000037999999999999995,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.862786874989979}
{"item_id":"senseval3.d001.s013.t002","gold_sense_keys":["acquire%2:40:00::"],"candidates":[{"index":1,"sense_key":"acquire%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"acquire%2:30:00::","synset_id":"assume.v.03"},{"index":3,"sense_key":"acquire%2:29:00::","synset_id":"grow.v.08"},{"index":4,"sense_key":"acquire%2:42:00::","synset_id":"acquire.v.04"},{"index":5,"sense_key":"acquire%2:40:06::","synset_id":"acquire.v.05"},{"index":6,"sense_key":"acquire%2:31:00::","synset_id":"learn.v.01"},{"index":7,"sense_key":"acquire%2:30:05::","synset_id":"develop.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"acquire%2:40:00::","call_ids":["senseval3.d001.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"acquire%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004032,"input_uncached_usd":0.0000384,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.953822584007867}
{"item_id":"senseval3.d001.s013.t003","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003158,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.260453583032358}
{"item_id":"senseval3.d001.s013.t004","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026304999999999997,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.034023290965706}
{"item_id":"senseval3.d001.s013.t007","gold_sense_keys":["prevent%2:41:00::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["senseval3.d001.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.000495,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.816543874971103}
{"item_id":"senseval3.d001.s013.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019055,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2531950830016285}
{"item_id":"senseval3.d001.s013.t010","gold_sense_keys":["bring_home_the_bacon%2:41:00::"],"candidates":[{"index":1,"sense_key":"bring_home_the_bacon%2:41:00::","synset_id":"succeed.v.01"},{"index":2,"sense_key":"bring_home_the_bacon%2:40:00::","synset_id":"provide.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bring_home_the_bacon%2:40:00::","call_ids":["senseval3.d001.s013.t010__v1__a1","senseval3.d001.s013.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bring_home_the_bacon%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1173,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00050665,"input_uncached_usd":0.00005865,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.180914417956956}
{"item_id":"senseval3.d001.s014.t001","gold_sense_keys":["come_to%2:42:03::"],"candidates":[{"index":1,"sense_key":"come_to%2:39:00::","synset_id":"hit.v.09"},{"index":2,"sense_key":"come_to%2:42:03::","synset_id":"refer.v.02"},{"index":3,"sense_key":"come_to%2:41:00::","synset_id":"come_to.v.03"},{"index":4,"sense_key":"come_to%2:29:00::","synset_id":"come_to.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come_to%2:42:03::","call_ids":["senseval3.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come_to%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008571,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.35684158402728}
{"item_id":"senseval3.d001.s014.t002","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029824999999999996,"input_uncached_usd":0.00003585,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.215663707989734}
{"item_id":"senseval3.d001.s014.t003","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018899999999999999,"input_uncached_usd":0.000029,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0769817500258796}
{"item_id":"senseval3.d001.s014.t004","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018999999999999998,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1236090830061585}
{"item_id":"senseval3.d001.s015.t000","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037124999999999994,"input_uncached_usd":0.00003205,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.037980124994647}
{"item_id":"senseval3.d001.s015.t002","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"},{"index":2,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002868,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.03446024999721}
{"item_id":"senseval3.d001.s015.t004","gold_sense_keys":["lean%2:42:01::"],"candidates":[{"index":1,"sense_key":"lean%2:38:00::","synset_id":"lean.v.01"},{"index":2,"sense_key":"lean%2:35:00::","synset_id":"lean.v.02"},{"index":3,"sense_key":"lean%2:42:01::","synset_id":"tend.v.01"},{"index":4,"sense_key":"lean%2:31:10::","synset_id":"lean.v.04"},{"index":5,"sense_key":"lean%2:38:07::","synset_id":"list.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"lean%2:42:01::","call_ids":["senseval3.d001.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"lean%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016534999999999998,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.605777832970489}
{"item_id":"senseval3.d001.s016.t000","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"president%1:18:04::","call_ids":["senseval3.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"president%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003436,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.11428145802347}
{"item_id":"senseval3.d001.s016.t001","gold_sense_keys":["limit%2:30:01::"],"candidates":[{"index":1,"sense_key":"limit%2:30:01::","synset_id":"restrict.v.03"},{"index":2,"sense_key":"limit%2:30:00::","synset_id":"limit.v.02"},{"index":3,"sense_key":"limit%2:32:00::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"limit%2:30:00::","call_ids":["senseval3.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"limit%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008257000000000001,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.760488917003386}
{"item_id":"senseval3.d001.s016.t002","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043995,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.568035541975405}
{"item_id":"senseval3.d001.s016.t003","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001824,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3284742499818094}
{"item_id":"senseval3.d001.s016.t005","gold_sense_keys":["particular%5:00:02:specific:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":3,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":5,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"particular%5:00:00:specific:00","call_ids":["senseval3.d001.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"particular%5:00:00:specific:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007310499999999999,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.333849415997975}
{"item_id":"senseval3.d001.s016.t008","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"president%1:18:04::","call_ids":["senseval3.d001.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"president%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003692,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.607310542021878}
{"item_id":"senseval3.d001.s016.t009","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"likely%3:00:04::","call_ids":["senseval3.d001.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"likely%3:00:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003143,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.5751121250214055}
{"item_id":"senseval3.d001.s016.t010","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"effective%5:00:00:operative:00","call_ids":["senseval3.d001.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"effective%5:00:00:operative:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029685,"input_uncached_usd":0.00003405,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.276234874967486}
{"item_id":"senseval3.d001.s016.t011","gold_sense_keys":["prevent%2:41:00::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["senseval3.d001.s016.t011__v1__a1","senseval3.d001.s016.t011__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1007,"cached_input_tokens":0,"output_tokens":3104,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.00129195,"input_uncached_usd":0.000050350000000000004,"input_cached_usd":0.0,"output_usd":0.0012415999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.137099498999305}
{"item_id":"senseval3.d001.s016.t012","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003147,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.230046709009912}
{"item_id":"senseval3.d001.s016.t016","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s016.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046554999999999996,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.21934037504252}
{"item_id":"senseval3.d001.s016.t017","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s016.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020794999999999999,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.741854332969524}
{"item_id":"senseval3.d001.s017.t000","gold_sense_keys":["individual%3:00:00::","individual%5:00:00:independent:00"],"candidates":[{"index":1,"sense_key":"individual%3:00:00::","synset_id":"individual.a.01"},{"index":2,"sense_key":"individual%5:00:00:independent:00","synset_id":"individual.s.01"},{"index":3,"sense_key":"individual%5:00:00:unshared:00","synset_id":"individual.s.02"},{"index":4,"sense_key":"individual%5:00:00:personal:00","synset_id":"individual.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%3:00:00::","call_ids":["senseval3.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008519,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.512077790976036}
{"item_id":"senseval3.d001.s017.t002","gold_sense_keys":["standing%1:26:00::"],"candidates":[{"index":1,"sense_key":"standing%1:26:00::","synset_id":"standing.n.01"},{"index":2,"sense_key":"standing%1:10:00::","synset_id":"standing.n.02"},{"index":3,"sense_key":"standing%1:04:00::","synset_id":"standing.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"standing%1:26:00::","call_ids":["senseval3.d001.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"standing%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043815,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0651309579843655}
{"item_id":"senseval3.d001.s017.t003","gold_sense_keys":["enhance%2:30:01::"],"candidates":[{"index":1,"sense_key":"enhance%2:30:00::","synset_id":"enhance.v.01"},{"index":2,"sense_key":"enhance%2:30:01::","synset_id":"enhance.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"enhance%2:30:01::","call_ids":["senseval3.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"enhance%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018069999999999997,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.321641082991846}
{"item_id":"senseval3.d001.s018.t001","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"simple%3:00:02::","call_ids":["senseval3.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"simple%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045,"input_uncached_usd":0.0000336,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.435605583013967}
{"item_id":"senseval3.d001.s018.t002","gold_sense_keys":["appear%2:39:00::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appear%2:30:01::","call_ids":["senseval3.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appear%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004261,"input_uncached_usd":0.0000357,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.82112145802239}
{"item_id":"senseval3.d001.s018.t003","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"explain%2:32:00::","call_ids":["senseval3.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"explain%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049235,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.633566582982894}
{"item_id":"senseval3.d001.s018.t004","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["senseval3.d001.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026179999999999997,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.252966083004139}
{"item_id":"senseval3.d001.s018.t005","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"thing%1:09:02::","call_ids":["senseval3.d001.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"thing%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005566,"input_uncached_usd":0.0000382,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.367473083024379}
{"item_id":"senseval3.d001.s019.t000","gold_sense_keys":["increase%2:30:00::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%2:30:00::","call_ids":["senseval3.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025925,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.090910833969247}
{"item_id":"senseval3.d001.s019.t001","gold_sense_keys":["take%2:30:01::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"take%2:30:01::","call_ids":["senseval3.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"take%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2239,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00045114999999999993,"input_uncached_usd":0.00011195,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.304768750036601}
{"item_id":"senseval3.d001.s019.t004","gold_sense_keys":["past%3:00:00::"],"candidates":[{"index":1,"sense_key":"past%3:00:00::","synset_id":"past.a.01"},{"index":2,"sense_key":"past%5:00:00:outgoing:00","synset_id":"past.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%3:00:00::","call_ids":["senseval3.d001.s019.t004__v1__a1","senseval3.d001.s019.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00052205,"input_uncached_usd":0.00004845,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.989899292064365}
{"item_id":"senseval3.d001.s019.t006","gold_sense_keys":["election%1:26:00::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028855,"input_uncached_usd":0.00002615,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.83940908301156}
{"item_id":"senseval3.d001.s019.t007","gold_sense_keys":["president%1:18:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"president%1:18:04::","call_ids":["senseval3.d001.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"president%1:18:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039559999999999997,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8773845420219}
{"item_id":"senseval3.d001.s019.t008","gold_sense_keys":["advent%1:04:00::"],"candidates":[{"index":1,"sense_key":"advent%1:04:00::","synset_id":"advent.n.01"},{"index":2,"sense_key":"advent%1:28:01::","synset_id":"advent.n.02"},{"index":3,"sense_key":"advent%1:11:00::","synset_id":"second_coming.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"advent%1:04:00::","call_ids":["senseval3.d001.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"advent%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003127,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.854132000007667}
{"item_id":"senseval3.d001.s019.t009","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"government%1:09:00::","call_ids":["senseval3.d001.s019.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"government%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023904999999999998,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.853041375055909}
{"item_id":"senseval3.d001.s019.t010","gold_sense_keys":["occupy%2:41:04::"],"candidates":[{"index":1,"sense_key":"occupy%2:41:00::","synset_id":"busy.v.01"},{"index":2,"sense_key":"occupy%2:42:00::","synset_id":"occupy.v.02"},{"index":3,"sense_key":"occupy%2:42:01::","synset_id":"occupy.v.03"},{"index":4,"sense_key":"occupy%2:42:04::","synset_id":"concern.v.02"},{"index":5,"sense_key":"occupy%2:33:00::","synset_id":"invade.v.01"},{"index":6,"sense_key":"occupy%2:40:08::","synset_id":"take.v.02"},{"index":7,"sense_key":"occupy%2:31:00::","synset_id":"absorb.v.09"},{"index":8,"sense_key":"occupy%2:41:04::","synset_id":"fill.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"occupy%2:41:04::","call_ids":["senseval3.d001.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"occupy%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024815,"input_uncached_usd":0.00003695,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8188027080032043}
{"item_id":"senseval3.d001.s019.t011","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["senseval3.d001.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039319999999999996,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.239558375033084}
{"item_id":"senseval3.d001.s019.t012","gold_sense_keys":["society%1:14:00::"],"candidates":[{"index":1,"sense_key":"society%1:14:00::","synset_id":"society.n.01"},{"index":2,"sense_key":"society%1:14:01::","synset_id":"club.n.02"},{"index":3,"sense_key":"society%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"society%1:14:02::","synset_id":"society.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"society%1:14:00::","call_ids":["senseval3.d001.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"society%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054555,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.898588625015691}
{"item_id":"senseval3.d001.s019.t013","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["senseval3.d001.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023605,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0102456659660675}
{"item_id":"senseval3.d001.s019.t016","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s019.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013869999999999998,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.576855625025928}
{"item_id":"senseval3.d001.s020.t000","gold_sense_keys":["explain%2:32:02::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"explain%2:32:02::","call_ids":["senseval3.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"explain%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026264999999999996,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.419012917031068}
{"item_id":"senseval3.d001.s020.t002","gold_sense_keys":["hold%2:31:01::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"hold%2:36:00::","call_ids":["senseval3.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"hold%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2009,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00067005,"input_uncached_usd":0.00010045,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.018980334047228}
{"item_id":"senseval3.d001.s020.t003","gold_sense_keys":["disdain%1:12:00::"],"candidates":[{"index":1,"sense_key":"disdain%1:12:00::","synset_id":"contempt.n.01"},{"index":2,"sense_key":"disdain%1:10:00::","synset_id":"condescension.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"disdain%1:12:00::","call_ids":["senseval3.d001.s020.t003__v1__a1","senseval3.d001.s020.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"disdain%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00062605,"input_uncached_usd":0.00005005,"input_cached_usd":0.0,"output_usd":0.0005759999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.538812750019133}
{"item_id":"senseval3.d001.s020.t004","gold_sense_keys":["love%2:37:00::"],"candidates":[{"index":1,"sense_key":"love%2:37:00::","synset_id":"love.v.01"},{"index":2,"sense_key":"love%2:37:02::","synset_id":"love.v.02"},{"index":3,"sense_key":"love%2:37:01::","synset_id":"love.v.03"},{"index":4,"sense_key":"love%2:35:00::","synset_id":"sleep_together.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"love%2:37:00::","call_ids":["senseval3.d001.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"love%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003407,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.33017029101029}
{"item_id":"senseval3.d001.s020.t007","gold_sense_keys":["representative%1:18:01::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"representative%1:18:01::","call_ids":["senseval3.d001.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"representative%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013634999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.312611375004053}
{"item_id":"senseval3.d001.s020.t008","gold_sense_keys":["individual%3:00:00::","individual%5:00:00:independent:00"],"candidates":[{"index":1,"sense_key":"individual%3:00:00::","synset_id":"individual.a.01"},{"index":2,"sense_key":"individual%5:00:00:independent:00","synset_id":"individual.s.01"},{"index":3,"sense_key":"individual%5:00:00:unshared:00","synset_id":"individual.s.02"},{"index":4,"sense_key":"individual%5:00:00:personal:00","synset_id":"individual.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"individual%5:00:00:personal:00","call_ids":["senseval3.d001.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"individual%5:00:00:personal:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":2704,"reasoning_output_tokens":2688},"cost":{"currency":"USD","total_usd":0.0011100499999999998,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0010816,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.357507375010755}
{"item_id":"senseval3.d001.s020.t010","gold_sense_keys":["constituent%1:18:00::"],"candidates":[{"index":1,"sense_key":"constituent%1:06:00::","synset_id":"component.n.03"},{"index":2,"sense_key":"constituent%1:18:00::","synset_id":"constituent.n.02"},{"index":3,"sense_key":"constituent%1:24:00::","synset_id":"part.n.01"},{"index":4,"sense_key":"constituent%1:10:00::","synset_id":"constituent.n.04"},{"index":5,"sense_key":"constituent%1:09:00::","synset_id":"component.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"constituent%1:18:00::","call_ids":["senseval3.d001.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"constituent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044985,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.063430042006075}
{"item_id":"senseval3.d001.s020.t011","gold_sense_keys":["appreciate%2:37:00::"],"candidates":[{"index":1,"sense_key":"appreciate%2:37:00::","synset_id":"appreciate.v.01"},{"index":2,"sense_key":"appreciate%2:31:00::","synset_id":"appreciate.v.02"},{"index":3,"sense_key":"appreciate%2:40:00::","synset_id":"prize.v.01"},{"index":4,"sense_key":"appreciate%2:30:00::","synset_id":"appreciate.v.04"},{"index":5,"sense_key":"appreciate%2:30:01::","synset_id":"appreciate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appreciate%2:40:00::","call_ids":["senseval3.d001.s020.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appreciate%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036799999999999995,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.35309554199921}
{"item_id":"senseval3.d001.s020.t012","gold_sense_keys":["specific%3:00:00::"],"candidates":[{"index":1,"sense_key":"specific%3:00:00::","synset_id":"specific.a.01"},{"index":2,"sense_key":"specific%5:00:00:specified:00","synset_id":"specific.s.01"},{"index":3,"sense_key":"specific%3:01:00::","synset_id":"specific.a.03"},{"index":4,"sense_key":"specific%3:00:02::","synset_id":"specific.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"specific%3:00:00::","call_ids":["senseval3.d001.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"specific%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021329999999999998,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.602991042018402}
{"item_id":"senseval3.d001.s020.t013","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s020.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023469999999999999,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.883704209001735}
{"item_id":"senseval3.d001.s020.t015","gold_sense_keys":["win%2:40:00::"],"candidates":[{"index":1,"sense_key":"win%2:33:00::","synset_id":"win.v.01"},{"index":2,"sense_key":"win%2:40:00::","synset_id":"acquire.v.05"},{"index":3,"sense_key":"win%2:33:01::","synset_id":"gain.v.05"},{"index":4,"sense_key":"win%2:41:04::","synset_id":"succeed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"win%2:33:01::","call_ids":["senseval3.d001.s020.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"win%2:33:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":2640,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.0010864499999999999,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0010559999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.312914333946537}
{"item_id":"senseval3.d001.s020.t016","gold_sense_keys":["overall%5:00:00:gross:00"],"candidates":[{"index":1,"sense_key":"overall%5:00:00:general:00","synset_id":"overall.s.01"},{"index":2,"sense_key":"overall%5:00:00:gross:00","synset_id":"overall.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overall%5:00:00:gross:00","call_ids":["senseval3.d001.s020.t016__v1__a1","senseval3.d001.s020.t016__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overall%5:00:00:gross:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":2656,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.0011090499999999999,"input_uncached_usd":0.000046649999999999996,"input_cached_usd":0.0,"output_usd":0.0010623999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.43050575000234}
{"item_id":"senseval3.d001.s020.t017","gold_sense_keys":["cost%1:07:01::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"},{"index":3,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cost%1:07:01::","call_ids":["senseval3.d001.s020.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cost%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018664999999999998,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0849677919759415}
{"item_id":"senseval3.d001.s020.t019","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s020.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004182,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.687660875031725}
{"item_id":"senseval3.d001.s020.t021","gold_sense_keys":["do%2:29:09::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d001.s020.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005933499999999999,"input_uncached_usd":0.00004895,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.11099929199554}
{"item_id":"senseval3.d001.s021.t001","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval3.d001.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021449999999999998,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.848085000005085}
{"item_id":"senseval3.d001.s021.t004","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024435,"input_uncached_usd":0.000032749999999999996,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.188836832996458}
{"item_id":"senseval3.d001.s021.t005","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002627,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.017904167005327}
{"item_id":"senseval3.d001.s021.t006","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s021.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041775,"input_uncached_usd":0.000026949999999999998,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.825499041995499}
{"item_id":"senseval3.d001.s021.t007","gold_sense_keys":["relative%5:00:00:proportionate:00"],"candidates":[{"index":1,"sense_key":"relative%3:00:00::","synset_id":"relative.a.01"},{"index":2,"sense_key":"relative%5:00:00:proportionate:00","synset_id":"proportional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relative%3:00:00::","call_ids":["senseval3.d001.s021.t007__v1__a1","senseval3.d001.s021.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relative%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1019,"cached_input_tokens":0,"output_tokens":2208,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.00093415,"input_uncached_usd":0.00005094999999999999,"input_cached_usd":0.0,"output_usd":0.0008832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.762258041009773}
{"item_id":"senseval3.d001.s021.t008","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044134999999999997,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.588606584002264}
{"item_id":"senseval3.d001.s021.t009","gold_sense_keys":["have%2:30:01::","have%2:40:00::","have%2:42:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"have%2:40:04::","call_ids":["senseval3.d001.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"have%2:40:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1224,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004772,"input_uncached_usd":0.0000612,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.384200959000736}
{"item_id":"senseval3.d001.s021.t011","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s021.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042635000000000004,"input_uncached_usd":0.00003555,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.356039500038605}
{"item_id":"senseval3.d001.s022.t002","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["senseval3.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014064999999999998,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.663809542020317}
{"item_id":"senseval3.d001.s022.t003","gold_sense_keys":["vulnerable%3:00:00::"],"candidates":[{"index":1,"sense_key":"vulnerable%3:00:00::","synset_id":"vulnerable.a.01"},{"index":2,"sense_key":"vulnerable%5:00:00:susceptible:00","synset_id":"vulnerable.s.01"},{"index":3,"sense_key":"vulnerable%5:00:00:weak:00","synset_id":"vulnerable.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"vulnerable%5:00:00:weak:00","call_ids":["senseval3.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"vulnerable%5:00:00:weak:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.000185,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6887465420295484}
{"item_id":"senseval3.d001.s022.t005","gold_sense_keys":["become%2:30:00::"],"candidates":[{"index":1,"sense_key":"become%2:30:00::","synset_id":"become.v.01"},{"index":2,"sense_key":"become%2:42:01::","synset_id":"become.v.02"},{"index":3,"sense_key":"become%2:42:00::","synset_id":"become.v.03"},{"index":4,"sense_key":"become%2:42:03::","synset_id":"become.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"become%2:42:01::","call_ids":["senseval3.d001.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"become%2:42:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036674999999999994,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.063923041976523}
{"item_id":"senseval3.d001.s022.t006","gold_sense_keys":["immune%5:00:00:unaffected:00"],"candidates":[{"index":1,"sense_key":"immune%3:01:00::","synset_id":"immune.a.01"},{"index":2,"sense_key":"immune%5:00:00:exempt:00","synset_id":"immune.s.01"},{"index":3,"sense_key":"immune%5:00:00:unsusceptible:00","synset_id":"immune.s.02"},{"index":4,"sense_key":"immune%5:00:00:unaffected:00","synset_id":"immune.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"immune%5:00:00:unaffected:00","call_ids":["senseval3.d001.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"immune%5:00:00:unaffected:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057085,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.378091124992352}
{"item_id":"senseval3.d001.s022.t007","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s022.t007__v1__a1","senseval3.d001.s022.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":985,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00054845,"input_uncached_usd":0.00004925,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.874087832984515}
{"item_id":"senseval3.d001.s023.t000","gold_sense_keys":["exception%1:09:01::"],"candidates":[{"index":1,"sense_key":"exception%1:09:00::","synset_id":"exception.n.01"},{"index":2,"sense_key":"exception%1:09:01::","synset_id":"exception.n.02"},{"index":3,"sense_key":"exception%1:10:00::","synset_id":"exception.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"exception%1:09:01::","call_ids":["senseval3.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"exception%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023749999999999997,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.979903833998833}
{"item_id":"senseval3.d001.s023.t002","gold_sense_keys":["trend%1:15:02::"],"candidates":[{"index":1,"sense_key":"trend%1:15:02::","synset_id":"tendency.n.04"},{"index":2,"sense_key":"trend%1:15:01::","synset_id":"course.n.03"},{"index":3,"sense_key":"trend%1:09:00::","synset_id":"drift.n.05"},{"index":4,"sense_key":"trend%1:09:02::","synset_id":"vogue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trend%1:09:00::","call_ids":["senseval3.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trend%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004199,"input_uncached_usd":0.0000295,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.978767999971751}
{"item_id":"senseval3.d001.s023.t003","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002867,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.484901209012605}
{"item_id":"senseval3.d001.s023.t004","gold_sense_keys":["freshman%1:18:01::"],"candidates":[{"index":1,"sense_key":"freshman%1:18:00::","synset_id":"freshman.n.01"},{"index":2,"sense_key":"freshman%1:18:01::","synset_id":"newcomer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"freshman%1:18:01::","call_ids":["senseval3.d001.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"freshman%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026044999999999996,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.493679625040386}
{"item_id":"senseval3.d001.s023.t005","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013365,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.804816916992422}
{"item_id":"senseval3.d001.s023.t006","gold_sense_keys":["bring%2:35:04::"],"candidates":[{"index":1,"sense_key":"bring%2:38:00::","synset_id":"bring.v.01"},{"index":2,"sense_key":"bring%2:35:04::","synset_id":"bring.v.02"},{"index":3,"sense_key":"bring%2:36:00::","synset_id":"bring.v.03"},{"index":4,"sense_key":"bring%2:35:00::","synset_id":"bring.v.04"},{"index":5,"sense_key":"bring%2:30:00::","synset_id":"bring.v.05"},{"index":6,"sense_key":"bring%2:35:02::","synset_id":"bring.v.06"},{"index":7,"sense_key":"bring%2:36:01::","synset_id":"institute.v.02"},{"index":8,"sense_key":"bring%2:40:02::","synset_id":"lend.v.01"},{"index":9,"sense_key":"bring%2:40:00::","synset_id":"fetch.v.02"},{"index":10,"sense_key":"bring%2:37:05::","synset_id":"bring.v.10"},{"index":11,"sense_key":"bring%2:32:03::","synset_id":"bring.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bring%2:35:04::","call_ids":["senseval3.d001.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bring%2:35:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00061375,"input_uncached_usd":0.000044149999999999996,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.14530570799252}
{"item_id":"senseval3.d001.s023.t007","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"office%1:04:02::","call_ids":["senseval3.d001.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"office%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005550999999999999,"input_uncached_usd":0.0000363,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.508396791992709}
{"item_id":"senseval3.d001.s023.t008","gold_sense_keys":["revolution%1:11:01::"],"candidates":[{"index":1,"sense_key":"revolution%1:11:01::","synset_id":"revolution.n.01"},{"index":2,"sense_key":"revolution%1:04:00::","synset_id":"revolution.n.02"},{"index":3,"sense_key":"revolution%1:11:00::","synset_id":"rotation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"revolution%1:11:01::","call_ids":["senseval3.d001.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"revolution%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016064999999999997,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5610710839973763}
{"item_id":"senseval3.d001.s023.t009","gold_sense_keys":["run%2:33:06::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"run%2:33:06::","call_ids":["senseval3.d001.s023.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"run%2:33:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2101,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00034225,"input_uncached_usd":0.00010504999999999999,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.2524097499554045}
{"item_id":"senseval3.d001.s024.t000","gold_sense_keys":["freshman%1:18:01::"],"candidates":[{"index":1,"sense_key":"freshman%1:18:00::","synset_id":"freshman.n.01"},{"index":2,"sense_key":"freshman%1:18:01::","synset_id":"newcomer.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"freshman%1:18:01::","call_ids":["senseval3.d001.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"freshman%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038879999999999996,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2080867500044405}
{"item_id":"senseval3.d001.s024.t001","gold_sense_keys":["place%2:35:00::"],"candidates":[{"index":1,"sense_key":"place%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"place%2:41:01::","synset_id":"place.v.02"},{"index":3,"sense_key":"place%2:31:01::","synset_id":"rate.v.01"},{"index":4,"sense_key":"place%2:40:00::","synset_id":"locate.v.03"},{"index":5,"sense_key":"place%2:32:01::","synset_id":"place.v.05"},{"index":6,"sense_key":"place%2:31:03::","synset_id":"place.v.06"},{"index":7,"sense_key":"place%2:33:03::","synset_id":"target.v.01"},{"index":8,"sense_key":"place%2:31:00::","synset_id":"identify.v.01"},{"index":9,"sense_key":"place%2:41:00::","synset_id":"place.v.09"},{"index":10,"sense_key":"place%2:36:00::","synset_id":"set.v.09"},{"index":11,"sense_key":"place%2:31:13::","synset_id":"place.v.11"},{"index":12,"sense_key":"place%2:42:00::","synset_id":"place.v.12"},{"index":13,"sense_key":"place%2:40:03::","synset_id":"invest.v.01"},{"index":14,"sense_key":"place%2:33:00::","synset_id":"station.v.01"},{"index":15,"sense_key":"place%2:33:02::","synset_id":"place.v.15"},{"index":16,"sense_key":"place%2:32:00::","synset_id":"place.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"place%2:35:00::","call_ids":["senseval3.d001.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"place%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1034,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005189,"input_uncached_usd":0.000051699999999999996,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.346584790968336}
{"item_id":"senseval3.d001.s024.t002","gold_sense_keys":["emphasis%1:07:00::","emphasis%1:26:00::"],"candidates":[{"index":1,"sense_key":"emphasis%1:26:00::","synset_id":"emphasis.n.01"},{"index":2,"sense_key":"emphasis%1:07:00::","synset_id":"vehemence.n.01"},{"index":3,"sense_key":"emphasis%1:10:01::","synset_id":"emphasis.n.03"},{"index":4,"sense_key":"emphasis%1:10:00::","synset_id":"stress.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"emphasis%1:10:01::","call_ids":["senseval3.d001.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"emphasis%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.42607304098783}
{"item_id":"senseval3.d001.s024.t004","gold_sense_keys":["role%1:04:00::","role%1:04:01::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["senseval3.d001.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057405,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.24374483403517}
{"item_id":"senseval3.d001.s024.t005","gold_sense_keys":["spread%2:32:00::","spread%2:35:00::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"spread%2:32:00::","call_ids":["senseval3.d001.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"spread%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005084,"input_uncached_usd":0.000040799999999999996,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.544088000024203}
{"item_id":"senseval3.d001.s024.t006","gold_sense_keys":["revolution%1:11:01::"],"candidates":[{"index":1,"sense_key":"revolution%1:11:01::","synset_id":"revolution.n.01"},{"index":2,"sense_key":"revolution%1:04:00::","synset_id":"revolution.n.02"},{"index":3,"sense_key":"revolution%1:11:00::","synset_id":"rotation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"revolution%1:11:01::","call_ids":["senseval3.d001.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"revolution%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023779999999999998,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.021066832996439}
{"item_id":"senseval3.d001.s024.t007","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.000245,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.012042834016029}
{"item_id":"senseval3.d001.s024.t008","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039174999999999995,"input_uncached_usd":0.000026949999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.3724096249789}
{"item_id":"senseval3.d001.s024.t009","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023914999999999999,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.357956791995093}
{"item_id":"senseval3.d001.s025.t002","gold_sense_keys":["indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":3,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":4,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"},{"index":5,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"indicate%2:32:01::","call_ids":["senseval3.d001.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"indicate%2:32:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029289999999999996,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.268399916996714}
{"item_id":"senseval3.d001.s025.t006","gold_sense_keys":["attract%2:35:00::"],"candidates":[{"index":1,"sense_key":"attract%2:35:00::","synset_id":"attract.v.01"},{"index":2,"sense_key":"attract%2:37:00::","synset_id":"attract.v.02"},{"index":3,"sense_key":"attract%2:35:01::","synset_id":"attract.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attract%2:37:00::","call_ids":["senseval3.d001.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attract%2:37:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002634,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.320031749957707}
{"item_id":"senseval3.d001.s025.t008","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s025.t008__v1__a1","senseval3.d001.s025.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00062255,"input_uncached_usd":0.00004654999999999999,"input_cached_usd":0.0,"output_usd":0.0005759999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.115947999991477}
{"item_id":"senseval3.d001.s025.t010","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"office%1:04:02::","call_ids":["senseval3.d001.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"office%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029715,"input_uncached_usd":0.00003475,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.602578666002955}
{"item_id":"senseval3.d001.s026.t000","gold_sense_keys":["potential%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"potential%3:00:00::","synset_id":"potential.a.01"},{"index":2,"sense_key":"potential%5:00:00:prospective:00","synset_id":"likely.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%5:00:00:prospective:00","call_ids":["senseval3.d001.s026.t000__v1__a1","senseval3.d001.s026.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%5:00:00:prospective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006987499999999999,"input_uncached_usd":0.00004594999999999999,"input_cached_usd":0.0,"output_usd":0.0006528,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.65396075003082}
{"item_id":"senseval3.d001.s026.t001","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s026.t001__v1__a1","senseval3.d001.s026.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":2528,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.00105685,"input_uncached_usd":0.00004565,"input_cached_usd":0.0,"output_usd":0.0010111999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.215201874962077}
{"item_id":"senseval3.d001.s026.t002","gold_sense_keys":["discourage%2:37:00::"],"candidates":[{"index":1,"sense_key":"discourage%2:32:01::","synset_id":"deter.v.01"},{"index":2,"sense_key":"discourage%2:37:00::","synset_id":"discourage.v.02"},{"index":3,"sense_key":"discourage%2:32:02::","synset_id":"warn.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discourage%2:37:00::","call_ids":["senseval3.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discourage%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036399999999999996,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.2740911659784615}
{"item_id":"senseval3.d001.s026.t003","gold_sense_keys":["run%2:33:06::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"run%2:33:06::","call_ids":["senseval3.d001.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"run%2:33:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2062,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00033989999999999997,"input_uncached_usd":0.0001031,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.26777008298086}
{"item_id":"senseval3.d001.s026.t006","gold_sense_keys":["prospect%1:09:01::"],"candidates":[{"index":1,"sense_key":"prospect%1:26:00::","synset_id":"prospect.n.01"},{"index":2,"sense_key":"prospect%1:09:01::","synset_id":"expectation.n.01"},{"index":3,"sense_key":"prospect%1:18:00::","synset_id":"candidate.n.02"},{"index":4,"sense_key":"prospect%1:09:00::","synset_id":"view.n.02"},{"index":5,"sense_key":"prospect%1:04:00::","synset_id":"prognosis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prospect%1:09:01::","call_ids":["senseval3.d001.s026.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prospect%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029025,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.563916541985236}
{"item_id":"senseval3.d001.s026.t007","gold_sense_keys":["defeat%1:11:00::"],"candidates":[{"index":1,"sense_key":"defeat%1:11:00::","synset_id":"defeat.n.01"},{"index":2,"sense_key":"defeat%1:12:00::","synset_id":"frustration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"defeat%1:11:00::","call_ids":["senseval3.d001.s026.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"defeat%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041275,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.359602583979722}
{"item_id":"senseval3.d001.s026.t009","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s026.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049225,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.410691500001121}
{"item_id":"senseval3.d001.s026.t010","gold_sense_keys":["opponent%1:18:02::"],"candidates":[{"index":1,"sense_key":"opponent%1:18:02::","synset_id":"opposition.n.04"},{"index":2,"sense_key":"opponent%1:18:00::","synset_id":"adversary.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"opponent%1:18:02::","call_ids":["senseval3.d001.s026.t010__v1__a1","senseval3.d001.s026.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"opponent%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00074935,"input_uncached_usd":0.00004535,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.057245582982432}
{"item_id":"senseval3.d001.s027.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013079999999999998,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.530125458026305}
{"item_id":"senseval3.d001.s027.t001","gold_sense_keys":["potential%5:00:00:prospective:00"],"candidates":[{"index":1,"sense_key":"potential%3:00:00::","synset_id":"potential.a.01"},{"index":2,"sense_key":"potential%5:00:00:prospective:00","synset_id":"likely.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"potential%5:00:00:prospective:00","call_ids":["senseval3.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"potential%5:00:00:prospective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046314999999999995,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.242299833043944}
{"item_id":"senseval3.d001.s027.t002","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s027.t002__v1__a1","senseval3.d001.s027.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006211499999999999,"input_uncached_usd":0.00004515,"input_cached_usd":0.0,"output_usd":0.000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.2653362909914}
{"item_id":"senseval3.d001.s027.t004","gold_sense_keys":["realize%2:31:00::","realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d001.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004999,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.305309874995146}
{"item_id":"senseval3.d001.s027.t007","gold_sense_keys":["work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%2:41:03::","call_ids":["senseval3.d001.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1533,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00044144999999999997,"input_uncached_usd":0.00007664999999999999,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.442745541979093}
{"item_id":"senseval3.d001.s027.t009","gold_sense_keys":["hinder%2:41:00::"],"candidates":[{"index":1,"sense_key":"hinder%2:41:01::","synset_id":"impede.v.01"},{"index":2,"sense_key":"hinder%2:41:00::","synset_id":"obstruct.v.01"},{"index":3,"sense_key":"hinder%2:33:00::","synset_id":"handicap.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hinder%2:41:00::","call_ids":["senseval3.d001.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hinder%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003384,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.400241624971386}
{"item_id":"senseval3.d001.s027.t010","gold_sense_keys":["attempt%1:04:00::"],"candidates":[{"index":1,"sense_key":"attempt%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"attempt%1:04:02::","synset_id":"attack.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attempt%1:04:00::","call_ids":["senseval3.d001.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attempt%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002854,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.108874041005038}
{"item_id":"senseval3.d001.s027.t012","gold_sense_keys":["slate%1:10:02::"],"candidates":[{"index":1,"sense_key":"slate%1:06:00::","synset_id":"slate.n.01"},{"index":2,"sense_key":"slate%1:27:01::","synset_id":"slate.n.02"},{"index":3,"sense_key":"slate%1:27:00::","synset_id":"slate.n.03"},{"index":4,"sense_key":"slate%1:10:02::","synset_id":"slate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"slate%1:10:02::","call_ids":["senseval3.d001.s027.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"slate%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031215,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.714385290979408}
{"item_id":"senseval3.d001.s027.t014","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s027.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030935,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.36948316701455}
{"item_id":"senseval3.d001.s028.t003","gold_sense_keys":["partial%5:00:00:incomplete:00"],"candidates":[{"index":1,"sense_key":"partial%5:00:00:incomplete:00","synset_id":"partial.s.01"},{"index":2,"sense_key":"partial%3:00:01::","synset_id":"partial.a.02"},{"index":3,"sense_key":"partial%5:00:00:inclined:02","synset_id":"fond.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"partial%5:00:00:incomplete:00","call_ids":["senseval3.d001.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"partial%5:00:00:incomplete:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002621,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.424611124966759}
{"item_id":"senseval3.d001.s028.t004","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reason%1:10:01::","call_ids":["senseval3.d001.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reason%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006799999999999999,"input_uncached_usd":0.0000336,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.569805042003281}
{"item_id":"senseval3.d001.s029.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033634999999999996,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.003245791012887}
{"item_id":"senseval3.d001.s029.t001","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023674999999999998,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.261427208024543}
{"item_id":"senseval3.d001.s029.t003","gold_sense_keys":["hold%2:40:00::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hold%2:40:00::","call_ids":["senseval3.d001.s029.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hold%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2000,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005928000000000001,"input_uncached_usd":0.00009999999999999999,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.998411541047972}
{"item_id":"senseval3.d001.s029.t005","gold_sense_keys":["share%1:04:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":3,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":4,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"share%1:04:00::","call_ids":["senseval3.d001.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"share%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003189,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.975672832981218}
{"item_id":"senseval3.d001.s029.t006","gold_sense_keys":["power%1:07:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"power%1:07:00::","call_ids":["senseval3.d001.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"power%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037679999999999994,"input_uncached_usd":0.0000376,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.812087667000014}
{"item_id":"senseval3.d001.s029.t007","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d001.s029.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044439999999999996,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1652545830002055}
{"item_id":"senseval3.d001.s029.t008","gold_sense_keys":["translate%2:30:01::"],"candidates":[{"index":1,"sense_key":"translate%2:32:00::","synset_id":"translate.v.01"},{"index":2,"sense_key":"translate%2:30:01::","synset_id":"translate.v.02"},{"index":3,"sense_key":"translate%2:31:00::","synset_id":"understand.v.03"},{"index":4,"sense_key":"translate%2:35:00::","synset_id":"translate.v.04"},{"index":5,"sense_key":"translate%2:30:00::","synset_id":"translate.v.05"},{"index":6,"sense_key":"translate%2:42:03::","synset_id":"translate.v.06"},{"index":7,"sense_key":"translate%2:42:00::","synset_id":"translate.v.07"},{"index":8,"sense_key":"translate%2:38:04::","synset_id":"translate.v.08"},{"index":9,"sense_key":"translate%2:32:06::","synset_id":"translate.v.09"},{"index":10,"sense_key":"translate%2:32:05::","synset_id":"translate.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"translate%2:42:03::","call_ids":["senseval3.d001.s029.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"translate%2:42:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035265,"input_uncached_usd":0.00003905,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.910495624993928}
{"item_id":"senseval3.d001.s029.t010","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s029.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002102,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1582455830066465}
{"item_id":"senseval3.d001.s029.t011","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["senseval3.d001.s029.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010624999999999998,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.246335582982283}
{"item_id":"senseval3.d001.s029.t015","gold_sense_keys":["have%2:40:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s029.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1209,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00019485,"input_uncached_usd":0.00006045,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9099580419715494}
{"item_id":"senseval3.d001.s029.t016","gold_sense_keys":["strong%3:00:00::"],"candidates":[{"index":1,"sense_key":"strong%3:00:00::","synset_id":"strong.a.01"},{"index":2,"sense_key":"strong%5:00:00:intense:00","synset_id":"strong.s.01"},{"index":3,"sense_key":"strong%5:00:00:powerful:00","synset_id":"potent.s.02"},{"index":4,"sense_key":"strong%3:00:04::","synset_id":"potent.a.03"},{"index":5,"sense_key":"strong%5:00:00:invulnerable:00","synset_id":"impregnable.s.01"},{"index":6,"sense_key":"strong%5:00:00:sound:01","synset_id":"solid.s.05"},{"index":7,"sense_key":"strong%5:00:00:irregular:00","synset_id":"strong.s.05"},{"index":8,"sense_key":"strong%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":9,"sense_key":"strong%5:00:00:fresh:01","synset_id":"strong.s.07"},{"index":10,"sense_key":"strong%5:00:00:forceful:00","synset_id":"firm.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strong%3:00:00::","call_ids":["senseval3.d001.s029.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strong%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":806,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048189999999999995,"input_uncached_usd":0.0000403,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.232021749950945}
{"item_id":"senseval3.d001.s029.t017","gold_sense_keys":["incentive%1:16:00::"],"candidates":[{"index":1,"sense_key":"incentive%1:16:00::","synset_id":"incentive.n.01"},{"index":2,"sense_key":"incentive%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"incentive%1:16:00::","call_ids":["senseval3.d001.s029.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"incentive%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046415,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.219091875012964}
{"item_id":"senseval3.d001.s029.t018","gold_sense_keys":["keep%2:41:02::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"keep%2:41:02::","call_ids":["senseval3.d001.s029.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"keep%2:41:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1304,"cached_input_tokens":0,"output_tokens":2128,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.0009163999999999999,"input_uncached_usd":0.0000652,"input_cached_usd":0.0,"output_usd":0.0008512,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.789522709033918}
{"item_id":"senseval3.d001.s029.t020","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s029.t020__v1__a1","senseval3.d001.s029.t020__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00039055,"input_uncached_usd":0.000044949999999999995,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.733282790984958}
{"item_id":"senseval3.d001.s029.t021","gold_sense_keys":["office%1:26:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"office%1:26:00::","call_ids":["senseval3.d001.s029.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"office%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034839999999999996,"input_uncached_usd":0.0000348,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5236228330177255}
{"item_id":"senseval3.d001.s030.t001","gold_sense_keys":["suggest%2:32:02::"],"candidates":[{"index":1,"sense_key":"suggest%2:32:00::","synset_id":"propose.v.01"},{"index":2,"sense_key":"suggest%2:32:04::","synset_id":"hint.v.01"},{"index":3,"sense_key":"suggest%2:32:02::","synset_id":"suggest.v.03"},{"index":4,"sense_key":"suggest%2:32:03::","synset_id":"indicate.v.05"},{"index":5,"sense_key":"suggest%2:32:01::","synset_id":"suggest.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"suggest%2:32:02::","call_ids":["senseval3.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"suggest%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018894999999999998,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.905667542014271}
{"item_id":"senseval3.d001.s030.t002","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010739999999999998,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.998563707980793}
{"item_id":"senseval3.d001.s030.t003","gold_sense_keys":["fare%2:42:00::"],"candidates":[{"index":1,"sense_key":"fare%2:42:00::","synset_id":"do.v.04"},{"index":2,"sense_key":"fare%2:34:00::","synset_id":"fare.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fare%2:42:00::","call_ids":["senseval3.d001.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fare%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005160000000000001,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.709305499971379}
{"item_id":"senseval3.d001.s030.t004","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["senseval3.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031855,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.550462457991671}
{"item_id":"senseval3.d001.s030.t005","gold_sense_keys":["campaign%1:11:00::"],"candidates":[{"index":1,"sense_key":"campaign%1:11:00::","synset_id":"political_campaign.n.01"},{"index":2,"sense_key":"campaign%1:04:02::","synset_id":"campaign.n.02"},{"index":3,"sense_key":"campaign%1:04:00::","synset_id":"campaign.n.03"},{"index":4,"sense_key":"campaign%1:04:01::","synset_id":"campaign.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"campaign%1:11:00::","call_ids":["senseval3.d001.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"campaign%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005471,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.765247000032105}
{"item_id":"senseval3.d001.s031.t000","gold_sense_keys":["local%3:01:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036399999999999996,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.302328208985273}
{"item_id":"senseval3.d001.s031.t003","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s031.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019219999999999998,"input_uncached_usd":0.0000322,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.091022917011287}
{"item_id":"senseval3.d001.s031.t004","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s031.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044175,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.086243249999825}
{"item_id":"senseval3.d001.s031.t005","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s031.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028879999999999997,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.211640125024132}
{"item_id":"senseval3.d001.s031.t007","gold_sense_keys":["chamber%1:14:00::"],"candidates":[{"index":1,"sense_key":"chamber%1:06:00::","synset_id":"chamber.n.01"},{"index":2,"sense_key":"chamber%1:08:00::","synset_id":"chamber.n.02"},{"index":3,"sense_key":"chamber%1:06:02::","synset_id":"chamber.n.03"},{"index":4,"sense_key":"chamber%1:14:00::","synset_id":"chamber.n.04"},{"index":5,"sense_key":"chamber%1:06:01::","synset_id":"bedroom.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"chamber%1:14:00::","call_ids":["senseval3.d001.s031.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"chamber%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001355,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.607678125030361}
{"item_id":"senseval3.d001.s031.t009","gold_sense_keys":["expect%2:31:00::"],"candidates":[{"index":1,"sense_key":"expect%2:31:00::","synset_id":"expect.v.01"},{"index":2,"sense_key":"expect%2:32:00::","synset_id":"ask.v.04"},{"index":3,"sense_key":"expect%2:31:01::","synset_id":"expect.v.03"},{"index":4,"sense_key":"expect%2:31:05::","synset_id":"expect.v.04"},{"index":5,"sense_key":"expect%2:29:02::","synset_id":"expect.v.05"},{"index":6,"sense_key":"expect%2:29:00::","synset_id":"have_a_bun_in_the_oven.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expect%2:31:00::","call_ids":["senseval3.d001.s031.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expect%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026869999999999997,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.5364552499959245}
{"item_id":"senseval3.d001.s031.t010","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s031.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002092,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.686388166039251}
{"item_id":"senseval3.d001.s031.t011","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"},{"index":2,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s031.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061795,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.970379917009268}
{"item_id":"senseval3.d001.s031.t012","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":2,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":3,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":4,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":5,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":6,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":7,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":8,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":9,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":10,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s031.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005323000000000001,"input_uncached_usd":0.0000395,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.954990917001851}
{"item_id":"senseval3.d001.s031.t013","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s031.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002104,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.897020625008736}
{"item_id":"senseval3.d001.s031.t015","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s031.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026254999999999996,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.805698875046801}
{"item_id":"senseval3.d001.s031.t016","gold_sense_keys":["counterpart%1:09:00::"],"candidates":[{"index":1,"sense_key":"counterpart%1:09:00::","synset_id":"counterpart.n.01"},{"index":2,"sense_key":"counterpart%1:06:00::","synset_id":"counterpart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"counterpart%1:09:00::","call_ids":["senseval3.d001.s031.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"counterpart%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005662,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.80403858300997}
{"item_id":"senseval3.d001.s032.t000","gold_sense_keys":["empirical%3:00:00::"],"candidates":[{"index":1,"sense_key":"empirical%3:00:00::","synset_id":"empirical.a.01"},{"index":2,"sense_key":"empirical%3:01:00::","synset_id":"empiric.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"empirical%3:00:00::","call_ids":["senseval3.d001.s032.t000__v1__a1","senseval3.d001.s032.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"empirical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041965,"input_uncached_usd":0.00004845,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.980630623991601}
{"item_id":"senseval3.d001.s032.t001","gold_sense_keys":["support%1:04:02::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"support%1:04:02::","call_ids":["senseval3.d001.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"support%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040934999999999995,"input_uncached_usd":0.00004455,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.995459917001426}
{"item_id":"senseval3.d001.s033.t000","gold_sense_keys":["piece%1:06:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"piece%1:06:05::","call_ids":["senseval3.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"piece%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":927,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006159500000000001,"input_uncached_usd":0.000046349999999999995,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.169858166016638}
{"item_id":"senseval3.d001.s033.t001","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval3.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00067085,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.188488166953903}
{"item_id":"senseval3.d001.s033.t004","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045224999999999996,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.703961084014736}
{"item_id":"senseval3.d001.s033.t005","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036444999999999994,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.094322708959226}
{"item_id":"senseval3.d001.s033.t007","gold_sense_keys":["willing%3:00:00::"],"candidates":[{"index":1,"sense_key":"willing%3:00:00::","synset_id":"willing.a.01"},{"index":2,"sense_key":"willing%5:00:00:voluntary:01","synset_id":"uncoerced.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"willing%3:00:00::","call_ids":["senseval3.d001.s033.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"willing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064305,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.575983499991708}
{"item_id":"senseval3.d001.s033.t008","gold_sense_keys":["engage%2:41:06::"],"candidates":[{"index":1,"sense_key":"engage%2:41:06::","synset_id":"prosecute.v.03"},{"index":2,"sense_key":"engage%2:31:00::","synset_id":"absorb.v.09"},{"index":3,"sense_key":"engage%2:41:01::","synset_id":"hire.v.01"},{"index":4,"sense_key":"engage%2:41:00::","synset_id":"engage.v.04"},{"index":5,"sense_key":"engage%2:32:00::","synset_id":"betroth.v.01"},{"index":6,"sense_key":"engage%2:30:00::","synset_id":"engage.v.06"},{"index":7,"sense_key":"engage%2:41:12::","synset_id":"engage.v.07"},{"index":8,"sense_key":"engage%2:40:01::","synset_id":"engage.v.08"},{"index":9,"sense_key":"engage%2:40:00::","synset_id":"lease.v.04"},{"index":10,"sense_key":"engage%2:35:00::","synset_id":"engage.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"engage%2:41:06::","call_ids":["senseval3.d001.s033.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"engage%2:41:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030139999999999995,"input_uncached_usd":0.000039,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1334135420038365}
{"item_id":"senseval3.d001.s033.t009","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"local%3:01:01::","call_ids":["senseval3.d001.s033.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"local%3:01:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002611,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.763088915962726}
{"item_id":"senseval3.d001.s033.t011","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s033.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013335,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2190152920084074}
{"item_id":"senseval3.d001.s033.t012","gold_sense_keys":["colleague%1:18:01::"],"candidates":[{"index":1,"sense_key":"colleague%1:18:00::","synset_id":"colleague.n.01"},{"index":2,"sense_key":"colleague%1:18:01::","synset_id":"colleague.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"colleague%1:18:00::","call_ids":["senseval3.d001.s033.t012__v1__a1","senseval3.d001.s033.t012__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"colleague%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":2464,"reasoning_output_tokens":2432},"cost":{"currency":"USD","total_usd":0.0010335499999999998,"input_uncached_usd":0.00004795,"input_cached_usd":0.0,"output_usd":0.0009855999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.928237584012095}
{"item_id":"senseval3.d001.s034.t000","gold_sense_keys":["first%4:02:00::"],"candidates":[{"index":1,"sense_key":"first%4:02:00::","synset_id":"first.r.01"},{"index":2,"sense_key":"first%4:02:02::","synset_id":"first.r.02"},{"index":3,"sense_key":"first%4:02:03::","synset_id":"first.r.03"},{"index":4,"sense_key":"first%4:02:01::","synset_id":"foremost.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"first%4:02:03::","call_ids":["senseval3.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"first%4:02:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000289,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.901461165980436}
{"item_id":"senseval3.d001.s034.t002","gold_sense_keys":["find%2:31:09::","find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find%2:32:00::","call_ids":["senseval3.d001.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":1231,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005508,"input_uncached_usd":0.000058399999999999997,"input_cached_usd":0.0,"output_usd":0.0004923999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.90700787503738}
{"item_id":"senseval3.d001.s034.t006","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00067305,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.335885540989693}
{"item_id":"senseval3.d001.s034.t007","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00065135,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.346658833033871}
{"item_id":"senseval3.d001.s034.t008","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"},{"index":2,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"budget%1:21:02::","call_ids":["senseval3.d001.s034.t008__v1__a1","senseval3.d001.s034.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047034999999999997,"input_uncached_usd":0.00004795,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.779129375005141}
{"item_id":"senseval3.d001.s034.t009","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s034.t009__v1__a1","senseval3.d001.s034.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041644999999999996,"input_uncached_usd":0.00004525,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.699546874966472}
{"item_id":"senseval3.d001.s035.t000","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s035.t000__v1__a1","senseval3.d001.s035.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00044445,"input_uncached_usd":0.00004764999999999999,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.145252001006156}
{"item_id":"senseval3.d001.s035.t001","gold_sense_keys":["primary%3:00:00::","primary%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"primary%3:00:00::","synset_id":"primary.a.01"},{"index":2,"sense_key":"primary%5:00:00:underived:00","synset_id":"primary.s.01"},{"index":3,"sense_key":"primary%5:00:00:important:00","synset_id":"chief.s.01"},{"index":4,"sense_key":"primary%5:00:00:basic:00","synset_id":"elementary.s.02"},{"index":5,"sense_key":"primary%5:00:00:essential:00","synset_id":"basal.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"primary%5:00:00:important:00","call_ids":["senseval3.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"primary%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023969999999999997,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.506066957954317}
{"item_id":"senseval3.d001.s035.t002","gold_sense_keys":["duty%1:04:02::"],"candidates":[{"index":1,"sense_key":"duty%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"duty%1:04:02::","synset_id":"duty.n.02"},{"index":3,"sense_key":"duty%1:21:00::","synset_id":"duty.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"duty%1:04:02::","call_ids":["senseval3.d001.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"duty%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028895,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.215019333001692}
{"item_id":"senseval3.d001.s035.t003","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s035.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044239999999999997,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.531575458007865}
{"item_id":"senseval3.d001.s035.t004","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002159,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3833090830012225}
{"item_id":"senseval3.d001.s035.t005","gold_sense_keys":["involve%2:42:00::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involve%2:42:00::","call_ids":["senseval3.d001.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involve%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00062715,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.073373415973037}
{"item_id":"senseval3.d001.s035.t006","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004405,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.226897542015649}
{"item_id":"senseval3.d001.s035.t008","gold_sense_keys":["indicate%2:32:01::","indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":3,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":4,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"},{"index":5,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indicate%2:32:02::","call_ids":["senseval3.d001.s035.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indicate%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039525,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.060694333980791}
{"item_id":"senseval3.d001.s035.t009","gold_sense_keys":["political%3:01:01::"],"candidates":[{"index":1,"sense_key":"political%3:00:00::","synset_id":"political.a.01"},{"index":2,"sense_key":"political%3:01:01::","synset_id":"political.a.02"},{"index":3,"sense_key":"political%3:01:02::","synset_id":"political.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"political%3:00:00::","call_ids":["senseval3.d001.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"political%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033805,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.16757258301368}
{"item_id":"senseval3.d001.s035.t010","gold_sense_keys":["philosophy%1:09:01::"],"candidates":[{"index":1,"sense_key":"philosophy%1:09:01::","synset_id":"doctrine.n.01"},{"index":2,"sense_key":"philosophy%1:09:00::","synset_id":"philosophy.n.02"},{"index":3,"sense_key":"philosophy%1:09:02::","synset_id":"philosophy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"philosophy%1:09:01::","call_ids":["senseval3.d001.s035.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"philosophy%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005438,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.059998832992278}
{"item_id":"senseval3.d001.s035.t011","gold_sense_keys":["lead%2:32:00::"],"candidates":[{"index":1,"sense_key":"lead%2:38:01::","synset_id":"lead.v.01"},{"index":2,"sense_key":"lead%2:42:04::","synset_id":"leave.v.07"},{"index":3,"sense_key":"lead%2:42:12::","synset_id":"lead.v.03"},{"index":4,"sense_key":"lead%2:38:00::","synset_id":"lead.v.04"},{"index":5,"sense_key":"lead%2:32:00::","synset_id":"lead.v.05"},{"index":6,"sense_key":"lead%2:42:00::","synset_id":"run.v.03"},{"index":7,"sense_key":"lead%2:41:00::","synset_id":"head.v.02"},{"index":8,"sense_key":"lead%2:42:01::","synset_id":"lead.v.08"},{"index":9,"sense_key":"lead%2:41:12::","synset_id":"contribute.v.03"},{"index":10,"sense_key":"lead%2:36:00::","synset_id":"conduct.v.02"},{"index":11,"sense_key":"lead%2:42:03::","synset_id":"go.v.25"},{"index":12,"sense_key":"lead%2:38:02::","synset_id":"precede.v.04"},{"index":13,"sense_key":"lead%2:42:02::","synset_id":"run.v.23"},{"index":14,"sense_key":"lead%2:32:01::","synset_id":"moderate.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"lead%2:32:00::","call_ids":["senseval3.d001.s035.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"lead%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036124999999999997,"input_uncached_usd":0.00004765,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.456887999956962}
{"item_id":"senseval3.d001.s035.t013","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"republican%1:17:00::","call_ids":["senseval3.d001.s035.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"republican%1:17:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004393,"input_uncached_usd":0.0000233,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.080492125009187}
{"item_id":"senseval3.d001.s035.t014","gold_sense_keys":["narrow%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"narrow%3:00:00::","synset_id":"narrow.a.01"},{"index":2,"sense_key":"narrow%5:00:00:limited:00","synset_id":"narrow.s.01"},{"index":3,"sense_key":"narrow%3:00:06::","synset_id":"narrow-minded.a.02"},{"index":4,"sense_key":"narrow%3:00:02::","synset_id":"narrow.a.04"},{"index":5,"sense_key":"narrow%5:00:00:careful:00","synset_id":"minute.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"narrow%5:00:00:limited:00","call_ids":["senseval3.d001.s035.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"narrow%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023899999999999998,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.023116208030842}
{"item_id":"senseval3.d001.s035.t015","gold_sense_keys":["constituent%1:18:00::"],"candidates":[{"index":1,"sense_key":"constituent%1:06:00::","synset_id":"component.n.03"},{"index":2,"sense_key":"constituent%1:18:00::","synset_id":"constituent.n.02"},{"index":3,"sense_key":"constituent%1:24:00::","synset_id":"part.n.01"},{"index":4,"sense_key":"constituent%1:10:00::","synset_id":"constituent.n.04"},{"index":5,"sense_key":"constituent%1:09:00::","synset_id":"component.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"constituent%1:18:00::","call_ids":["senseval3.d001.s035.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"constituent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029529999999999997,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.674251333985012}
{"item_id":"senseval3.d001.s035.t016","gold_sense_keys":["concern%1:09:00::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concern%1:09:00::","call_ids":["senseval3.d001.s035.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concern%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036989999999999994,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.622229458997026}
{"item_id":"senseval3.d001.s036.t002","gold_sense_keys":["assumption%1:09:00::","assumption%1:10:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assumption%1:10:00::","call_ids":["senseval3.d001.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assumption%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045214999999999996,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.228310250036884}
{"item_id":"senseval3.d001.s036.t003","gold_sense_keys":["valid%3:00:00::"],"candidates":[{"index":1,"sense_key":"valid%3:00:00::","synset_id":"valid.a.01"},{"index":2,"sense_key":"valid%5:00:00:unexpired:00","synset_id":"valid.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"valid%3:00:00::","call_ids":["senseval3.d001.s036.t003__v1__a1","senseval3.d001.s036.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"valid%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00044235,"input_uncached_usd":0.000045549999999999996,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.006538000074215}
{"item_id":"senseval3.d001.s036.t004","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s036.t004__v1__a1","senseval3.d001.s036.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":883,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038974999999999996,"input_uncached_usd":0.00004415,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.161785874981433}
{"item_id":"senseval3.d001.s036.t005","gold_sense_keys":["have%2:30:01::","have%2:40:00::","have%2:42:00::","have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["senseval3.d001.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1201,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00039965,"input_uncached_usd":0.000060049999999999996,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0819574579945765}
{"item_id":"senseval3.d001.s036.t006","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s036.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":2321,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009519999999999999,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0009283999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.685195917030796}
{"item_id":"senseval3.d001.s036.t007","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"},{"index":2,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021299999999999997,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6242930839653127}
{"item_id":"senseval3.d001.s036.t009","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"vote%1:04:01::","call_ids":["senseval3.d001.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"vote%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":3921,"reasoning_output_tokens":3904},"cost":{"currency":"USD","total_usd":0.00159835,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.0015684,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":24.470036083017476}
{"item_id":"senseval3.d001.s036.t010","gold_sense_keys":["republican%1:18:01::"],"candidates":[{"index":1,"sense_key":"republican%1:18:01::","synset_id":"republican.n.01"},{"index":2,"sense_key":"republican%1:18:00::","synset_id":"republican.n.02"},{"index":3,"sense_key":"republican%1:17:00::","synset_id":"republican.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%1:18:01::","call_ids":["senseval3.d001.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013179999999999998,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5672839170438237}
{"item_id":"senseval3.d001.s036.t011","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["senseval3.d001.s036.t011__v1__a1","senseval3.d001.s036.t011__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004950499999999999,"input_uncached_usd":0.00004704999999999999,"input_cached_usd":0.0,"output_usd":0.000448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.959235208982136}
{"item_id":"senseval3.d001.s036.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"vote%1:09:00::","call_ids":["senseval3.d001.s036.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"vote%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047154999999999994,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.773074875003658}
{"item_id":"senseval3.d001.s036.t014","gold_sense_keys":["reflect%2:39:01::"],"candidates":[{"index":1,"sense_key":"reflect%2:39:01::","synset_id":"reflect.v.01"},{"index":2,"sense_key":"reflect%2:31:00::","synset_id":"chew_over.v.01"},{"index":3,"sense_key":"reflect%2:39:00::","synset_id":"reflect.v.03"},{"index":4,"sense_key":"reflect%2:43:00::","synset_id":"reflect.v.04"},{"index":5,"sense_key":"reflect%2:39:02::","synset_id":"reflect.v.05"},{"index":6,"sense_key":"reflect%2:32:01::","synset_id":"reflect.v.06"},{"index":7,"sense_key":"reflect%2:32:00::","synset_id":"reflect.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reflect%2:39:01::","call_ids":["senseval3.d001.s036.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reflect%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052505,"input_uncached_usd":0.00003225,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.532871207979042}
{"item_id":"senseval3.d001.s036.t015","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s036.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002428,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.505721791996621}
{"item_id":"senseval3.d001.s036.t016","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s036.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064555,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.443794374994468}
{"item_id":"senseval3.d001.s036.t017","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s036.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004673999999999999,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.325173208955675}
{"item_id":"senseval3.d001.s036.t020","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s036.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042035,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.667168415966444}
{"item_id":"senseval3.d001.s036.t021","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["senseval3.d001.s036.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037769999999999997,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.141817791969515}
{"item_id":"senseval3.d001.s036.t023","gold_sense_keys":["devote%2:40:00::"],"candidates":[{"index":1,"sense_key":"devote%2:32:00::","synset_id":"give.v.18"},{"index":2,"sense_key":"devote%2:31:00::","synset_id":"give.v.10"},{"index":3,"sense_key":"devote%2:40:00::","synset_id":"devote.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"devote%2:40:00::","call_ids":["senseval3.d001.s036.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"devote%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005176,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.169004875002429}
{"item_id":"senseval3.d001.s036.t024","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s036.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002866,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.574997457966674}
{"item_id":"senseval3.d001.s037.t001","gold_sense_keys":["indicate%2:32:01::","indicate%2:32:02::"],"candidates":[{"index":1,"sense_key":"indicate%2:32:01::","synset_id":"bespeak.v.01"},{"index":2,"sense_key":"indicate%2:32:03::","synset_id":"indicate.v.02"},{"index":3,"sense_key":"indicate%2:32:00::","synset_id":"indicate.v.03"},{"index":4,"sense_key":"indicate%2:32:02::","synset_id":"argue.v.03"},{"index":5,"sense_key":"indicate%2:32:04::","synset_id":"indicate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indicate%2:32:02::","call_ids":["senseval3.d001.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indicate%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031675000000000003,"input_uncached_usd":0.000028749999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8052264580037445}
{"item_id":"senseval3.d001.s037.t002","gold_sense_keys":["case%1:26:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["senseval3.d001.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005276,"input_uncached_usd":0.000059999999999999995,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5010077499900945}
{"item_id":"senseval3.d001.s038.t000","gold_sense_keys":["democratic%3:01:00::"],"candidates":[{"index":1,"sense_key":"democratic%3:00:00::","synset_id":"democratic.a.01"},{"index":2,"sense_key":"democratic%3:01:00::","synset_id":"democratic.a.02"},{"index":3,"sense_key":"democratic%5:00:00:common:01","synset_id":"democratic.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democratic%3:01:00::","call_ids":["senseval3.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democratic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002622,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.269685291976202}
{"item_id":"senseval3.d001.s038.t001","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043985,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.299893792020157}
{"item_id":"senseval3.d001.s038.t002","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"},{"index":2,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026445,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.473124957992695}
{"item_id":"senseval3.d001.s038.t004","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s038.t004__v1__a1","senseval3.d001.s038.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1035,"cached_input_tokens":0,"output_tokens":1057,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047455,"input_uncached_usd":0.00005175,"input_cached_usd":0.0,"output_usd":0.0004228,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.718657083983999}
{"item_id":"senseval3.d001.s038.t005","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"attendance%1:28:00::","call_ids":["senseval3.d001.s038.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"attendance%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064465,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.072040750004817}
{"item_id":"senseval3.d001.s038.t006","gold_sense_keys":["rate%1:24:00::"],"candidates":[{"index":1,"sense_key":"rate%1:28:00::","synset_id":"rate.n.01"},{"index":2,"sense_key":"rate%1:21:00::","synset_id":"rate.n.02"},{"index":3,"sense_key":"rate%1:07:00::","synset_id":"pace.n.03"},{"index":4,"sense_key":"rate%1:24:00::","synset_id":"rate.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rate%1:24:00::","call_ids":["senseval3.d001.s038.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rate%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021324999999999998,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.1211679999832995}
{"item_id":"senseval3.d001.s039.t000","gold_sense_keys":["show%2:32:00::","show%2:32:03::","show%2:36:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"show%2:39:00::","call_ids":["senseval3.d001.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"show%2:39:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005166,"input_uncached_usd":0.000049399999999999995,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.07368404197041}
{"item_id":"senseval3.d001.s039.t001","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"table%1:14:00::","call_ids":["senseval3.d001.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019329999999999998,"input_uncached_usd":0.000033299999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.402766416023951}
{"item_id":"senseval3.d001.s039.t002","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023475,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.20904841599986}
{"item_id":"senseval3.d001.s039.t004","gold_sense_keys":["proportion%1:24:01::"],"candidates":[{"index":1,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":2,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":3,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":4,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":5,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"proportion%1:24:01::","call_ids":["senseval3.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"proportion%1:24:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00078155,"input_uncached_usd":0.000032749999999999996,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.099994707969017}
{"item_id":"senseval3.d001.s039.t005","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057275,"input_uncached_usd":0.000028749999999999997,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.478466667002067}
{"item_id":"senseval3.d001.s039.t006","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037189999999999993,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.298195832991041}
{"item_id":"senseval3.d001.s039.t008","gold_sense_keys":["office%1:06:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"office%1:04:01::","call_ids":["senseval3.d001.s039.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"office%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":1681,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007095499999999999,"input_uncached_usd":0.000037149999999999995,"input_cached_usd":0.0,"output_usd":0.0006724,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.928952332993504}
{"item_id":"senseval3.d001.s039.t009","gold_sense_keys":["local%3:00:01::"],"candidates":[{"index":1,"sense_key":"local%3:00:01::","synset_id":"local.a.01"},{"index":2,"sense_key":"local%3:01:01::","synset_id":"local.a.02"},{"index":3,"sense_key":"local%3:00:02::","synset_id":"local.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"local%3:00:01::","call_ids":["senseval3.d001.s039.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"local%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049445,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.573966708034277}
{"item_id":"senseval3.d001.s039.t011","gold_sense_keys":["duty%1:04:02::"],"candidates":[{"index":1,"sense_key":"duty%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"duty%1:04:02::","synset_id":"duty.n.02"},{"index":3,"sense_key":"duty%1:21:00::","synset_id":"duty.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"duty%1:04:02::","call_ids":["senseval3.d001.s039.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"duty%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024055,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.671814374974929}
{"item_id":"senseval3.d001.s039.t013","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["senseval3.d001.s039.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029665,"input_uncached_usd":0.00003425,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.720198250026442}
{"item_id":"senseval3.d001.s039.t014","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["senseval3.d001.s039.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":1809,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007509999999999999,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.0007235999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.04545891698217}
{"item_id":"senseval3.d001.s039.t015","gold_sense_keys":["making%1:04:00::"],"candidates":[{"index":1,"sense_key":"making%1:04:00::","synset_id":"devising.n.01"},{"index":2,"sense_key":"making%1:07:00::","synset_id":"qualification.n.01"},{"index":3,"sense_key":"making%1:06:00::","synset_id":"making.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"making%1:04:00::","call_ids":["senseval3.d001.s039.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"making%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039325,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.620567583013326}
{"item_id":"senseval3.d001.s039.t016","gold_sense_keys":["activity%1:04:00::"],"candidates":[{"index":1,"sense_key":"activity%1:04:00::","synset_id":"activity.n.01"},{"index":2,"sense_key":"activity%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"activity%1:22:00::","synset_id":"bodily_process.n.01"},{"index":4,"sense_key":"activity%1:26:01::","synset_id":"activity.n.04"},{"index":5,"sense_key":"activity%1:22:01::","synset_id":"natural_process.n.01"},{"index":6,"sense_key":"activity%1:07:00::","synset_id":"activeness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"activity%1:26:00::","call_ids":["senseval3.d001.s039.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"activity%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034714999999999995,"input_uncached_usd":0.000033549999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.529565292003099}
{"item_id":"senseval3.d001.s039.t018","gold_sense_keys":["relative%5:00:00:proportionate:00"],"candidates":[{"index":1,"sense_key":"relative%3:00:00::","synset_id":"relative.a.01"},{"index":2,"sense_key":"relative%5:00:00:proportionate:00","synset_id":"proportional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"relative%5:00:00:proportionate:00","call_ids":["senseval3.d001.s039.t018__v1__a1","senseval3.d001.s039.t018__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"relative%5:00:00:proportionate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1079,"cached_input_tokens":0,"output_tokens":3681,"reasoning_output_tokens":3648},"cost":{"currency":"USD","total_usd":0.0015263499999999999,"input_uncached_usd":0.00005395,"input_cached_usd":0.0,"output_usd":0.0014724,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":23.750511792022735}
{"item_id":"senseval3.d001.s039.t019","gold_sense_keys":["office%1:06:00::"],"candidates":[{"index":1,"sense_key":"office%1:06:00::","synset_id":"office.n.01"},{"index":2,"sense_key":"office%1:14:01::","synset_id":"agency.n.01"},{"index":3,"sense_key":"office%1:04:01::","synset_id":"function.n.03"},{"index":4,"sense_key":"office%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"office%1:14:00::","synset_id":"office.n.05"},{"index":6,"sense_key":"office%1:04:00::","synset_id":"office.n.06"},{"index":7,"sense_key":"office%1:04:02::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"office%1:14:01::","call_ids":["senseval3.d001.s039.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"office%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006323,"input_uncached_usd":0.0000371,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.430330624978524}
{"item_id":"senseval3.d001.s040.t000","gold_sense_keys":["examination%1:04:00::"],"candidates":[{"index":1,"sense_key":"examination%1:04:00::","synset_id":"examination.n.01"},{"index":2,"sense_key":"examination%1:10:00::","synset_id":"examination.n.02"},{"index":3,"sense_key":"examination%1:10:01::","synset_id":"interrogation.n.03"},{"index":4,"sense_key":"examination%1:09:03::","synset_id":"examen.n.01"},{"index":5,"sense_key":"examination%1:04:02::","synset_id":"examination.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"examination%1:04:00::","call_ids":["senseval3.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"examination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032105,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.683163499983493}
{"item_id":"senseval3.d001.s040.t001","gold_sense_keys":["change%1:04:00::","change%1:11:00::","change%1:19:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%1:11:00::","call_ids":["senseval3.d001.s040.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":892,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005118,"input_uncached_usd":0.0000446,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5840953750303015}
{"item_id":"senseval3.d001.s040.t002","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006263,"input_uncached_usd":0.0000311,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.530462040973362}
{"item_id":"senseval3.d001.s040.t003","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["senseval3.d001.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006038500000000001,"input_uncached_usd":0.00003425,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.145012374967337}
{"item_id":"senseval3.d001.s040.t004","gold_sense_keys":["control%1:04:00::","control%1:07:00::","control%1:24:00::"],"candidates":[{"index":1,"sense_key":"control%1:07:00::","synset_id":"control.n.01"},{"index":2,"sense_key":"control%1:24:00::","synset_id":"control.n.02"},{"index":3,"sense_key":"control%1:04:02::","synset_id":"control.n.03"},{"index":4,"sense_key":"control%1:09:02::","synset_id":"control_condition.n.01"},{"index":5,"sense_key":"control%1:04:00::","synset_id":"control.n.05"},{"index":6,"sense_key":"control%1:26:00::","synset_id":"dominance.n.02"},{"index":7,"sense_key":"control%1:07:01::","synset_id":"restraint.n.02"},{"index":8,"sense_key":"control%1:09:01::","synset_id":"command.n.06"},{"index":9,"sense_key":"control%1:06:00::","synset_id":"control.n.09"},{"index":10,"sense_key":"control%1:18:00::","synset_id":"control.n.10"},{"index":11,"sense_key":"control%1:10:00::","synset_id":"control.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"control%1:07:00::","call_ids":["senseval3.d001.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"control%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.00095065,"input_uncached_usd":0.00004825,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.244604041974526}
{"item_id":"senseval3.d001.s040.t005","gold_sense_keys":["body%1:14:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"body%1:14:00::","call_ids":["senseval3.d001.s040.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"body%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":914,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005128999999999999,"input_uncached_usd":0.0000457,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.1939512499957345}
{"item_id":"senseval3.d001.s040.t007","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s040.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003213,"input_uncached_usd":0.000033299999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.62435183301568}
{"item_id":"senseval3.d001.s040.t008","gold_sense_keys":["reveal%2:39:00::"],"candidates":[{"index":1,"sense_key":"reveal%2:39:00::","synset_id":"uncover.v.01"},{"index":2,"sense_key":"reveal%2:32:00::","synset_id":"unwrap.v.02"},{"index":3,"sense_key":"reveal%2:39:04::","synset_id":"reveal.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"reveal%2:32:00::","call_ids":["senseval3.d001.s040.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"reveal%2:32:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044744999999999995,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.940632333047688}
{"item_id":"senseval3.d001.s040.t009","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["senseval3.d001.s040.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004215,"input_uncached_usd":0.0000311,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.535283749981318}
{"item_id":"senseval3.d001.s040.t010","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"difference%1:11:00::","call_ids":["senseval3.d001.s040.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"difference%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":675,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024494999999999996,"input_uncached_usd":0.00003375,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.131762875011191}
{"item_id":"senseval3.d001.s040.t011","gold_sense_keys":["note%2:32:00::"],"candidates":[{"index":1,"sense_key":"note%2:32:00::","synset_id":"note.v.01"},{"index":2,"sense_key":"note%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"note%2:39:02::","synset_id":"note.v.03"},{"index":4,"sense_key":"note%2:32:01::","synset_id":"note.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"note%2:32:00::","call_ids":["senseval3.d001.s040.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"note%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009343,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.34448666597018}
{"item_id":"senseval3.d001.s040.t012","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"table%1:14:00::","call_ids":["senseval3.d001.s040.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037489999999999995,"input_uncached_usd":0.0000357,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9544580829679035}
{"item_id":"senseval3.d001.s040.t013","gold_sense_keys":["attribute%2:31:00::"],"candidates":[{"index":1,"sense_key":"attribute%2:31:00::","synset_id":"impute.v.01"},{"index":2,"sense_key":"attribute%2:31:01::","synset_id":"assign.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attribute%2:31:00::","call_ids":["senseval3.d001.s040.t013__v1__a1","senseval3.d001.s040.t013__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attribute%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1165,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006598499999999999,"input_uncached_usd":0.00005824999999999999,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.050377498962916}
{"item_id":"senseval3.d001.s040.t015","gold_sense_keys":["control%1:07:00::"],"candidates":[{"index":1,"sense_key":"control%1:07:00::","synset_id":"control.n.01"},{"index":2,"sense_key":"control%1:24:00::","synset_id":"control.n.02"},{"index":3,"sense_key":"control%1:04:02::","synset_id":"control.n.03"},{"index":4,"sense_key":"control%1:09:02::","synset_id":"control_condition.n.01"},{"index":5,"sense_key":"control%1:04:00::","synset_id":"control.n.05"},{"index":6,"sense_key":"control%1:26:00::","synset_id":"dominance.n.02"},{"index":7,"sense_key":"control%1:07:01::","synset_id":"restraint.n.02"},{"index":8,"sense_key":"control%1:09:01::","synset_id":"command.n.06"},{"index":9,"sense_key":"control%1:06:00::","synset_id":"control.n.09"},{"index":10,"sense_key":"control%1:18:00::","synset_id":"control.n.10"},{"index":11,"sense_key":"control%1:10:00::","synset_id":"control.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"control%1:26:00::","call_ids":["senseval3.d001.s040.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"control%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046425,"input_uncached_usd":0.00004825,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.27362566604279}
{"item_id":"senseval3.d001.s040.t016","gold_sense_keys":["democrat%1:18:01::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"democrat%1:18:01::","call_ids":["senseval3.d001.s040.t016__v1__a1","senseval3.d001.s040.t016__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"democrat%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00042534999999999996,"input_uncached_usd":0.00005415,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.352222709043417}
{"item_id":"senseval3.d001.s040.t017","gold_sense_keys":["exercise%2:34:00::"],"candidates":[{"index":1,"sense_key":"exercise%2:34:00::","synset_id":"exert.v.01"},{"index":2,"sense_key":"exercise%2:41:00::","synset_id":"practice.v.01"},{"index":3,"sense_key":"exercise%2:29:01::","synset_id":"exercise.v.03"},{"index":4,"sense_key":"exercise%2:29:00::","synset_id":"exercise.v.04"},{"index":5,"sense_key":"exercise%2:31:00::","synset_id":"drill.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exercise%2:34:00::","call_ids":["senseval3.d001.s040.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exercise%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037219999999999994,"input_uncached_usd":0.000032999999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.700830665999092}
{"item_id":"senseval3.d001.s040.t019","gold_sense_keys":["status%1:26:00::"],"candidates":[{"index":1,"sense_key":"status%1:26:00::","synset_id":"status.n.01"},{"index":2,"sense_key":"status%1:26:01::","synset_id":"condition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"status%1:26:00::","call_ids":["senseval3.d001.s040.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"status%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004695999999999999,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.015093834022991}
{"item_id":"senseval3.d001.s040.t020","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s040.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029434999999999997,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.159986332990229}
{"item_id":"senseval3.d001.s040.t021","gold_sense_keys":["resource%1:07:00::"],"candidates":[{"index":1,"sense_key":"resource%1:21:00::","synset_id":"resource.n.01"},{"index":2,"sense_key":"resource%1:07:00::","synset_id":"resource.n.02"},{"index":3,"sense_key":"resource%1:09:00::","synset_id":"resource.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"resource%1:07:00::","call_ids":["senseval3.d001.s040.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"resource%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026565,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.542134791961871}
{"item_id":"senseval3.d001.s040.t022","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["senseval3.d001.s040.t022__v1__a1","senseval3.d001.s040.t022__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1133,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00063265,"input_uncached_usd":0.000056649999999999995,"input_cached_usd":0.0,"output_usd":0.000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.69544445804786}
{"item_id":"senseval3.d001.s040.t023","gold_sense_keys":["staff%1:14:01::"],"candidates":[{"index":1,"sense_key":"staff%1:14:01::","synset_id":"staff.n.01"},{"index":2,"sense_key":"staff%1:06:00::","synset_id":"staff.n.02"},{"index":3,"sense_key":"staff%1:14:00::","synset_id":"staff.n.03"},{"index":4,"sense_key":"staff%1:27:00::","synset_id":"staff.n.04"},{"index":5,"sense_key":"staff%1:10:01::","synset_id":"staff.n.05"},{"index":6,"sense_key":"staff%1:10:00::","synset_id":"staff.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"staff%1:14:01::","call_ids":["senseval3.d001.s040.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"staff%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00068135,"input_uncached_usd":0.000034949999999999996,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.728753458999563}
{"item_id":"senseval3.d001.s041.t001","gold_sense_keys":["piece%1:06:00::"],"candidates":[{"index":1,"sense_key":"piece%1:06:00::","synset_id":"piece.n.01"},{"index":2,"sense_key":"piece%1:06:05::","synset_id":"piece.n.02"},{"index":3,"sense_key":"piece%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"piece%1:10:01::","synset_id":"musical_composition.n.01"},{"index":5,"sense_key":"piece%1:11:00::","synset_id":"piece.n.05"},{"index":6,"sense_key":"piece%1:10:02::","synset_id":"piece.n.06"},{"index":7,"sense_key":"piece%1:06:03::","synset_id":"firearm.n.01"},{"index":8,"sense_key":"piece%1:13:00::","synset_id":"piece.n.08"},{"index":9,"sense_key":"piece%1:07:00::","synset_id":"piece.n.09"},{"index":10,"sense_key":"piece%1:06:04::","synset_id":"objet_d'art.n.01"},{"index":11,"sense_key":"piece%1:28:00::","synset_id":"while.n.01"},{"index":12,"sense_key":"piece%1:21:00::","synset_id":"slice.n.01"},{"index":13,"sense_key":"piece%1:06:02::","synset_id":"man.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"piece%1:06:00::","call_ids":["senseval3.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"piece%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1021,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00049265,"input_uncached_usd":0.00005105,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.309745249978732}
{"item_id":"senseval3.d001.s041.t002","gold_sense_keys":["evidence%1:09:00::","evidence%1:10:01::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evidence%1:09:00::","call_ids":["senseval3.d001.s041.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evidence%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036874999999999994,"input_uncached_usd":0.000029149999999999997,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.20234412496211}
{"item_id":"senseval3.d001.s041.t003","gold_sense_keys":["hold%2:42:00::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hold%2:42:00::","call_ids":["senseval3.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hold%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2088,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0005204,"input_uncached_usd":0.00010439999999999999,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.694131915981416}
{"item_id":"senseval3.d001.s041.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024294999999999997,"input_uncached_usd":0.00003175,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.529441000020597}
{"item_id":"senseval3.d001.s041.t005","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"factor%1:09:01::","call_ids":["senseval3.d001.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"factor%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045434999999999996,"input_uncached_usd":0.00003835,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.996744083007798}
{"item_id":"senseval3.d001.s041.t006","gold_sense_keys":["constant%5:00:00:invariable:00"],"candidates":[{"index":1,"sense_key":"constant%5:00:00:invariable:00","synset_id":"changeless.s.01"},{"index":2,"sense_key":"constant%3:00:00::","synset_id":"constant.a.02"},{"index":3,"sense_key":"constant%5:00:00:continuous:01","synset_id":"ceaseless.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constant%5:00:00:invariable:00","call_ids":["senseval3.d001.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constant%5:00:00:invariable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":722,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003196,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.00028879999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.369771874975413}
{"item_id":"senseval3.d001.s041.t007","gold_sense_keys":["incumbency%1:04:00::","incumbency%1:28:00::"],"candidates":[{"index":1,"sense_key":"incumbency%1:28:00::","synset_id":"tenure.n.01"},{"index":2,"sense_key":"incumbency%1:04:01::","synset_id":"incumbency.n.02"},{"index":3,"sense_key":"incumbency%1:04:00::","synset_id":"incumbency.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"incumbency%1:04:00::","call_ids":["senseval3.d001.s041.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"incumbency%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044385,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.976800750009716}
{"item_id":"senseval3.d001.s041.t008","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"advantage%1:07:00::","call_ids":["senseval3.d001.s041.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"advantage%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024004999999999998,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.567179750010837}
{"item_id":"senseval3.d001.s041.t009","gold_sense_keys":["regional%5:00:00:territorial:00"],"candidates":[{"index":1,"sense_key":"regional%3:01:00::","synset_id":"regional.a.01"},{"index":2,"sense_key":"regional%5:00:00:territorial:00","synset_id":"regional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"regional%3:01:00::","call_ids":["senseval3.d001.s041.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"regional%3:01:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005188500000000001,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.982124290952925}
{"item_id":"senseval3.d001.s041.t010","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"factor%1:09:01::","call_ids":["senseval3.d001.s041.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"factor%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037749999999999996,"input_uncached_usd":0.000038299999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.309831457969267}
{"item_id":"senseval3.d001.s041.t011","gold_sense_keys":["difference%1:23:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"difference%1:23:00::","call_ids":["senseval3.d001.s041.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"difference%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002703,"input_uncached_usd":0.0000335,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.793608416977804}
{"item_id":"senseval3.d001.s041.t012","gold_sense_keys":["popular%5:00:00:democratic:00"],"candidates":[{"index":1,"sense_key":"popular%3:00:00::","synset_id":"popular.a.01"},{"index":2,"sense_key":"popular%5:00:00:democratic:00","synset_id":"popular.s.01"},{"index":3,"sense_key":"popular%5:00:01:common:01","synset_id":"democratic.s.01"},{"index":4,"sense_key":"popular%5:00:00:nonclassical:00","synset_id":"popular.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"popular%5:00:00:democratic:00","call_ids":["senseval3.d001.s041.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"popular%5:00:00:democratic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":642,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007297,"input_uncached_usd":0.0000321,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.562090040999465}
{"item_id":"senseval3.d001.s041.t013","gold_sense_keys":["vote%1:04:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s041.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052755,"input_uncached_usd":0.00003475,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.884747874981258}
{"item_id":"senseval3.d001.s041.t014","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s041.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004708499999999999,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.151284584018867}
{"item_id":"senseval3.d001.s041.t015","gold_sense_keys":["presidential%3:01:00::"],"candidates":[{"index":1,"sense_key":"presidential%3:01:00::","synset_id":"presidential.a.01"},{"index":2,"sense_key":"presidential%3:00:00::","synset_id":"presidential.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"presidential%3:01:00::","call_ids":["senseval3.d001.s041.t015__v1__a1","senseval3.d001.s041.t015__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"presidential%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":1185,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00052925,"input_uncached_usd":0.00005525,"input_cached_usd":0.0,"output_usd":0.0004739999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.357835292932577}
{"item_id":"senseval3.d001.s041.t017","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s041.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016094999999999998,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.508783959026914}
{"item_id":"senseval3.d001.s041.t018","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004842499999999999,"input_uncached_usd":0.00004265,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.671427374996711}
{"item_id":"senseval3.d001.s041.t019","gold_sense_keys":["conduct%2:41:00::"],"candidates":[{"index":1,"sense_key":"conduct%2:41:00::","synset_id":"conduct.v.01"},{"index":2,"sense_key":"conduct%2:36:00::","synset_id":"conduct.v.02"},{"index":3,"sense_key":"conduct%2:41:01::","synset_id":"behave.v.02"},{"index":4,"sense_key":"conduct%2:38:01::","synset_id":"lead.v.01"},{"index":5,"sense_key":"conduct%2:38:00::","synset_id":"impart.v.03"},{"index":6,"sense_key":"conduct%2:36:04::","synset_id":"conduct.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conduct%2:41:00::","call_ids":["senseval3.d001.s041.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conduct%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047899999999999993,"input_uncached_usd":0.0000374,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.3317611659877}
{"item_id":"senseval3.d001.s041.t020","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["senseval3.d001.s041.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004719999999999999,"input_uncached_usd":0.0000304,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5998563329922035}
{"item_id":"senseval3.d001.s041.t021","gold_sense_keys":["turn_out%2:42:00::","turn_out%2:42:01::"],"candidates":[{"index":1,"sense_key":"turn_out%2:42:00::","synset_id":"prove.v.01"},{"index":2,"sense_key":"turn_out%2:42:01::","synset_id":"turn_out.v.02"},{"index":3,"sense_key":"turn_out%2:36:01::","synset_id":"turn_out.v.03"},{"index":4,"sense_key":"turn_out%2:42:02::","synset_id":"turn_out.v.04"},{"index":5,"sense_key":"turn_out%2:30:00::","synset_id":"turn_out.v.05"},{"index":6,"sense_key":"turn_out%2:36:00::","synset_id":"bear.v.05"},{"index":7,"sense_key":"turn_out%2:35:00::","synset_id":"eject.v.01"},{"index":8,"sense_key":"turn_out%2:41:03::","synset_id":"turn_out.v.08"},{"index":9,"sense_key":"turn_out%2:40:03::","synset_id":"turn_out.v.09"},{"index":10,"sense_key":"turn_out%2:38:00::","synset_id":"turn_out.v.10"},{"index":11,"sense_key":"turn_out%2:35:01::","synset_id":"switch_off.v.01"},{"index":12,"sense_key":"turn_out%2:29:00::","synset_id":"get_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"turn_out%2:42:01::","call_ids":["senseval3.d001.s041.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"turn_out%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006703999999999999,"input_uncached_usd":0.000049199999999999997,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.371684417012148}
{"item_id":"senseval3.d001.s041.t022","gold_sense_keys":["positive%5:00:00:plus:00"],"candidates":[{"index":1,"sense_key":"positive%3:00:01::","synset_id":"positive.a.01"},{"index":2,"sense_key":"positive%5:00:00:certain:02","synset_id":"convinced.s.01"},{"index":3,"sense_key":"positive%5:00:00:advantageous:00","synset_id":"plus.s.01"},{"index":4,"sense_key":"positive%3:00:04::","synset_id":"positive.a.04"},{"index":5,"sense_key":"positive%5:00:00:formal:01","synset_id":"positive.s.03"},{"index":6,"sense_key":"positive%5:00:00:undeniable:00","synset_id":"incontrovertible.s.01"},{"index":7,"sense_key":"positive%3:01:00::","synset_id":"positivist.a.01"},{"index":8,"sense_key":"positive%3:00:05::","synset_id":"positive.a.08"},{"index":9,"sense_key":"positive%5:00:00:plus:00","synset_id":"positive.s.05"},{"index":10,"sense_key":"positive%5:00:00:charged:00","synset_id":"positive.s.06"},{"index":11,"sense_key":"positive%5:00:00:confident:00","synset_id":"cocksure.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"positive%3:00:05::","call_ids":["senseval3.d001.s041.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"positive%3:00:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":943,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005655500000000001,"input_uncached_usd":0.00004715,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.786731166008394}
{"item_id":"senseval3.d001.s041.t023","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"function%1:24:01::","call_ids":["senseval3.d001.s041.t023__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"function%1:24:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00053275,"input_uncached_usd":0.000039949999999999995,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.970585959032178}
{"item_id":"senseval3.d001.s041.t027","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["senseval3.d001.s041.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047224999999999996,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.221716667001601}
{"item_id":"senseval3.d001.s041.t028","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["senseval3.d001.s041.t028__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050985,"input_uncached_usd":0.00004265,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.6628389590187}
{"item_id":"senseval3.d001.s041.t029","gold_sense_keys":["progressive%3:00:02::"],"candidates":[{"index":1,"sense_key":"progressive%3:00:01::","synset_id":"progressive.a.01"},{"index":2,"sense_key":"progressive%5:00:00:liberal:00","synset_id":"progressive.s.01"},{"index":3,"sense_key":"progressive%3:00:02::","synset_id":"progressive.a.03"},{"index":4,"sense_key":"progressive%5:00:00:increasing:00","synset_id":"progressive.s.02"},{"index":5,"sense_key":"progressive%5:00:00:ordered:00","synset_id":"progressive.s.03"},{"index":6,"sense_key":"progressive%5:00:00:active:02","synset_id":"progressive.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"progressive%3:00:02::","call_ids":["senseval3.d001.s041.t029__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"progressive%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002451,"input_uncached_usd":0.0000339,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.959093416982796}
{"item_id":"senseval3.d001.s041.t031","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d001.s041.t031__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1199,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047635,"input_uncached_usd":0.00005995,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.419610166980419}
{"item_id":"senseval3.d001.s041.t032","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["senseval3.d001.s041.t032__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035055,"input_uncached_usd":0.00003695,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.89374291698914}
{"item_id":"senseval3.d001.s041.t034","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:00::","call_ids":["senseval3.d001.s041.t034__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00084265,"input_uncached_usd":0.00004265,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.252047374960966}
{"item_id":"senseval3.d001.s042.t000","gold_sense_keys":["put%2:32:00::"],"candidates":[{"index":1,"sense_key":"put%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"put%2:35:05::","synset_id":"put.v.02"},{"index":3,"sense_key":"put%2:32:00::","synset_id":"frame.v.04"},{"index":4,"sense_key":"put%2:34:00::","synset_id":"put.v.04"},{"index":5,"sense_key":"put%2:40:00::","synset_id":"invest.v.01"},{"index":6,"sense_key":"put%2:31:13::","synset_id":"place.v.11"},{"index":7,"sense_key":"put%2:39:13::","synset_id":"put.v.07"},{"index":8,"sense_key":"put%2:36:13::","synset_id":"put.v.08"},{"index":9,"sense_key":"put%2:31:00::","synset_id":"arrange.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"put%2:32:00::","call_ids":["senseval3.d001.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"put%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0002002,"input_uncached_usd":0.0000402,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.145339249982499}
{"item_id":"senseval3.d001.s042.t001","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s042.t001__v1__a1","senseval3.d001.s042.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00057715,"input_uncached_usd":0.00005235,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.787555541028269}
{"item_id":"senseval3.d001.s042.t002","gold_sense_keys":["president%1:04:04::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"president%1:04:04::","call_ids":["senseval3.d001.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"president%1:04:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":3152,"reasoning_output_tokens":3136},"cost":{"currency":"USD","total_usd":0.0012943,"input_uncached_usd":0.0000335,"input_cached_usd":0.0,"output_usd":0.0012607999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.951603958033957}
{"item_id":"senseval3.d001.s042.t003","gold_sense_keys":["look_on%2:31:00::"],"candidates":[{"index":1,"sense_key":"look_on%2:39:00::","synset_id":"watch.v.04"},{"index":2,"sense_key":"look_on%2:31:00::","synset_id":"think_of.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look_on%2:31:00::","call_ids":["senseval3.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look_on%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00062175,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.926397417031694}
{"item_id":"senseval3.d001.s042.t005","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s042.t005__v1__a1","senseval3.d001.s042.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1155,"cached_input_tokens":0,"output_tokens":1505,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00065975,"input_uncached_usd":0.000057749999999999994,"input_cached_usd":0.0,"output_usd":0.000602,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.63901258300757}
{"item_id":"senseval3.d001.s042.t006","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s042.t006__v1__a1","senseval3.d001.s042.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1047,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006795499999999999,"input_uncached_usd":0.00005235,"input_cached_usd":0.0,"output_usd":0.0006272,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.394462832948193}
{"item_id":"senseval3.d001.s042.t009","gold_sense_keys":["severe%5:00:02:nonindulgent:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":2,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":3,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":4,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":5,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":6,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"severe%5:00:01:bad:00","call_ids":["senseval3.d001.s042.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"severe%5:00:01:bad:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00060605,"input_uncached_usd":0.000036049999999999995,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.66243508295156}
{"item_id":"senseval3.d001.s043.t000","gold_sense_keys":["appear%2:39:00::"],"candidates":[{"index":1,"sense_key":"appear%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"appear%2:30:00::","synset_id":"appear.v.02"},{"index":3,"sense_key":"appear%2:30:01::","synset_id":"appear.v.03"},{"index":4,"sense_key":"appear%2:39:01::","synset_id":"appear.v.04"},{"index":5,"sense_key":"appear%2:30:02::","synset_id":"appear.v.05"},{"index":6,"sense_key":"appear%2:36:00::","synset_id":"appear.v.06"},{"index":7,"sense_key":"appear%2:41:03::","synset_id":"appear.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"appear%2:39:01::","call_ids":["senseval3.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"appear%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050625,"input_uncached_usd":0.00003905,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.9009461250389}
{"item_id":"senseval3.d001.s043.t001","gold_sense_keys":["take%2:30:01::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"take%2:30:01::","call_ids":["senseval3.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"take%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2298,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0007613,"input_uncached_usd":0.0001149,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.098768417024985}
{"item_id":"senseval3.d001.s043.t002","gold_sense_keys":["same%3:00:00::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"same%3:00:00::","call_ids":["senseval3.d001.s043.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"same%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042025,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.298602499999106}
{"item_id":"senseval3.d001.s043.t007","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["senseval3.d001.s043.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066065,"input_uncached_usd":0.00003985,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.229558541963343}
{"item_id":"senseval3.d001.s043.t008","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"},{"index":4,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s043.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.000551,"input_uncached_usd":0.0000322,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.678494208026677}
{"item_id":"senseval3.d001.s043.t009","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"level%1:26:00::","call_ids":["senseval3.d001.s043.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"level%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003278,"input_uncached_usd":0.0000398,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.533212291018572}
{"item_id":"senseval3.d001.s044.t003","gold_sense_keys":["line%1:09:00::"],"candidates":[{"index":1,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":2,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":3,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":4,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":5,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":6,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":7,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":8,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":9,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":10,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":11,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":12,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":13,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":14,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":15,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":16,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":17,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":18,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":19,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":20,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":21,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":22,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":23,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":24,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":25,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":26,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":27,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":28,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":29,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":30,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":29,"chosen_sense_key":"line%1:09:00::","call_ids":["senseval3.d001.s044.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":29,"predicted_sense_key":"line%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1716,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0005273999999999999,"input_uncached_usd":0.0000858,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.185530791990459}
{"item_id":"senseval3.d001.s044.t004","gold_sense_keys":["reverse%1:24:00::"],"candidates":[{"index":1,"sense_key":"reverse%1:24:00::","synset_id":"reverse.n.01"},{"index":2,"sense_key":"reverse%1:06:00::","synset_id":"reverse.n.02"},{"index":3,"sense_key":"reverse%1:11:01::","synset_id":"reverse.n.03"},{"index":4,"sense_key":"reverse%1:06:02::","synset_id":"reverse.n.04"},{"index":5,"sense_key":"reverse%1:04:00::","synset_id":"reverse.n.05"},{"index":6,"sense_key":"reverse%1:04:01::","synset_id":"reversion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reverse%1:24:00::","call_ids":["senseval3.d001.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reverse%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032315,"input_uncached_usd":0.00003515,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.321161708969157}
{"item_id":"senseval3.d001.s045.t001","gold_sense_keys":["investigation%1:04:00::"],"candidates":[{"index":1,"sense_key":"investigation%1:09:00::","synset_id":"probe.n.01"},{"index":2,"sense_key":"investigation%1:04:00::","synset_id":"investigation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"investigation%1:09:00::","call_ids":["senseval3.d001.s045.t001__v1__a1","senseval3.d001.s045.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"investigation%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1115,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00050375,"input_uncached_usd":0.00005575,"input_cached_usd":0.0,"output_usd":0.000448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.022460667009}
{"item_id":"senseval3.d001.s045.t003","gold_sense_keys":["hold%2:42:00::"],"candidates":[{"index":1,"sense_key":"hold%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"hold%2:35:00::","synset_id":"hold.v.02"},{"index":3,"sense_key":"hold%2:36:00::","synset_id":"hold.v.03"},{"index":4,"sense_key":"hold%2:40:00::","synset_id":"have.v.01"},{"index":5,"sense_key":"hold%2:31:01::","synset_id":"deem.v.01"},{"index":6,"sense_key":"hold%2:37:00::","synset_id":"harbor.v.01"},{"index":7,"sense_key":"hold%2:35:03::","synset_id":"restrain.v.03"},{"index":8,"sense_key":"hold%2:40:02::","synset_id":"retain.v.03"},{"index":9,"sense_key":"hold%2:40:04::","synset_id":"bear.v.11"},{"index":10,"sense_key":"hold%2:35:01::","synset_id":"hold.v.10"},{"index":11,"sense_key":"hold%2:42:13::","synset_id":"hold.v.11"},{"index":12,"sense_key":"hold%2:42:05::","synset_id":"accommodate.v.04"},{"index":13,"sense_key":"hold%2:42:01::","synset_id":"hold.v.13"},{"index":14,"sense_key":"hold%2:35:02::","synset_id":"hold.v.14"},{"index":15,"sense_key":"hold%2:42:02::","synset_id":"prevail.v.02"},{"index":16,"sense_key":"hold%2:31:08::","synset_id":"hold.v.16"},{"index":17,"sense_key":"hold%2:42:07::","synset_id":"hold.v.17"},{"index":18,"sense_key":"hold%2:42:14::","synset_id":"contain.v.05"},{"index":19,"sense_key":"hold%2:41:00::","synset_id":"reserve.v.04"},{"index":20,"sense_key":"hold%2:33:05::","synset_id":"defend.v.03"},{"index":21,"sense_key":"hold%2:32:13::","synset_id":"oblige.v.02"},{"index":22,"sense_key":"hold%2:31:04::","synset_id":"hold.v.22"},{"index":23,"sense_key":"hold%2:31:10::","synset_id":"hold.v.23"},{"index":24,"sense_key":"hold%2:42:08::","synset_id":"defy.v.01"},{"index":25,"sense_key":"hold%2:42:04::","synset_id":"apply.v.02"},{"index":26,"sense_key":"hold%2:42:12::","synset_id":"hold.v.26"},{"index":27,"sense_key":"hold%2:41:15::","synset_id":"control.v.02"},{"index":28,"sense_key":"hold%2:41:13::","synset_id":"hold.v.28"},{"index":29,"sense_key":"hold%2:41:14::","synset_id":"hold.v.29"},{"index":30,"sense_key":"hold%2:38:05::","synset_id":"halt.v.01"},{"index":31,"sense_key":"hold%2:35:09::","synset_id":"hold.v.31"},{"index":32,"sense_key":"hold%2:34:00::","synset_id":"carry.v.33"},{"index":33,"sense_key":"hold%2:33:00::","synset_id":"hold.v.33"},{"index":34,"sense_key":"hold%2:32:11::","synset_id":"declare.v.04"},{"index":35,"sense_key":"hold%2:32:15::","synset_id":"agree.v.01"},{"index":36,"sense_key":"hold%2:29:04::","synset_id":"hold.v.36"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hold%2:42:00::","call_ids":["senseval3.d001.s045.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hold%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2087,"cached_input_tokens":0,"output_tokens":1746,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00080275,"input_uncached_usd":0.00010434999999999999,"input_cached_usd":0.0,"output_usd":0.0006984,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.12469362502452}
{"item_id":"senseval3.d001.s045.t004","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["senseval3.d001.s045.t004__v1__a1","senseval3.d001.s045.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1315,"cached_input_tokens":0,"output_tokens":3104,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.0013073499999999999,"input_uncached_usd":0.00006575,"input_cached_usd":0.0,"output_usd":0.0012415999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":22.150089165952522}
{"item_id":"senseval3.d001.s045.t005","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"factor%1:09:01::","call_ids":["senseval3.d001.s045.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"factor%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027509999999999996,"input_uncached_usd":0.000038299999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.827649083046708}
{"item_id":"senseval3.d001.s045.t006","gold_sense_keys":["constant%5:00:00:invariable:00"],"candidates":[{"index":1,"sense_key":"constant%5:00:00:invariable:00","synset_id":"changeless.s.01"},{"index":2,"sense_key":"constant%3:00:00::","synset_id":"constant.a.02"},{"index":3,"sense_key":"constant%5:00:00:continuous:01","synset_id":"ceaseless.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"constant%5:00:00:invariable:00","call_ids":["senseval3.d001.s045.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"constant%5:00:00:invariable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024194999999999997,"input_uncached_usd":0.000030749999999999995,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.300250125001185}
{"item_id":"senseval3.d001.s045.t007","gold_sense_keys":["difference%1:23:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"difference%1:23:00::","call_ids":["senseval3.d001.s045.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"difference%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055185,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.464959708973765}
{"item_id":"senseval3.d001.s045.t008","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:00::","call_ids":["senseval3.d001.s045.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006122500000000001,"input_uncached_usd":0.00004265,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.991713540977798}
{"item_id":"senseval3.d001.s045.t009","gold_sense_keys":["major%3:00:01::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"major%3:00:06::","call_ids":["senseval3.d001.s045.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"major%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":914,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040175,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.0003656,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.202408499957528}
{"item_id":"senseval3.d001.s045.t010","gold_sense_keys":["party%1:14:01::"],"candidates":[{"index":1,"sense_key":"party%1:14:01::","synset_id":"party.n.01"},{"index":2,"sense_key":"party%1:14:00::","synset_id":"party.n.02"},{"index":3,"sense_key":"party%1:14:02::","synset_id":"party.n.03"},{"index":4,"sense_key":"party%1:11:00::","synset_id":"party.n.04"},{"index":5,"sense_key":"party%1:18:00::","synset_id":"party.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"party%1:14:01::","call_ids":["senseval3.d001.s045.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"party%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019304999999999998,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5118173329974525}
{"item_id":"senseval3.d001.s045.t011","gold_sense_keys":["vote%1:09:00::"],"candidates":[{"index":1,"sense_key":"vote%1:04:00::","synset_id":"vote.n.01"},{"index":2,"sense_key":"vote%1:04:01::","synset_id":"vote.n.02"},{"index":3,"sense_key":"vote%1:07:00::","synset_id":"right_to_vote.n.01"},{"index":4,"sense_key":"vote%1:14:00::","synset_id":"vote.n.04"},{"index":5,"sense_key":"vote%1:09:00::","synset_id":"vote.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vote%1:04:00::","call_ids":["senseval3.d001.s045.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vote%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":694,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034829999999999996,"input_uncached_usd":0.000034699999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.818066833016928}
{"item_id":"senseval3.d001.s045.t012","gold_sense_keys":["go%2:42:06::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"go%2:42:06::","call_ids":["senseval3.d001.s045.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"go%2:42:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1858,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0005348999999999999,"input_uncached_usd":0.0000929,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.802199375000782}
{"item_id":"senseval3.d001.s045.t013","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s045.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005988,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.726230499974918}
{"item_id":"senseval3.d001.s045.t015","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["senseval3.d001.s045.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028885,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.018441708001774}
{"item_id":"senseval3.d001.s045.t016","gold_sense_keys":["republican%3:01:01::"],"candidates":[{"index":1,"sense_key":"republican%3:01:01::","synset_id":"republican.a.01"},{"index":2,"sense_key":"republican%5:00:00:democratic:00","synset_id":"republican.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"republican%3:01:01::","call_ids":["senseval3.d001.s045.t016__v1__a1","senseval3.d001.s045.t016__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"republican%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1215,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00050875,"input_uncached_usd":0.00006075,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.10336454201024}
{"item_id":"senseval3.d001.s045.t017","gold_sense_keys":["share%1:04:00::"],"candidates":[{"index":1,"sense_key":"share%1:21:00::","synset_id":"share.n.01"},{"index":2,"sense_key":"share%1:21:01::","synset_id":"share.n.02"},{"index":3,"sense_key":"share%1:04:00::","synset_id":"parcel.n.02"},{"index":4,"sense_key":"share%1:04:01::","synset_id":"contribution.n.01"},{"index":5,"sense_key":"share%1:06:00::","synset_id":"plowshare.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"share%1:04:00::","call_ids":["senseval3.d001.s045.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"share%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006304500000000001,"input_uncached_usd":0.000035249999999999996,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.589406374958344}
{"item_id":"senseval3.d001.s045.t019","gold_sense_keys":["house%1:14:02::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"house%1:14:02::","call_ids":["senseval3.d001.s045.t019__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"house%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":934,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003603,"input_uncached_usd":0.0000467,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.618225959013216}
{"item_id":"senseval3.d001.s045.t020","gold_sense_keys":["positive%5:00:00:plus:00"],"candidates":[{"index":1,"sense_key":"positive%3:00:01::","synset_id":"positive.a.01"},{"index":2,"sense_key":"positive%5:00:00:certain:02","synset_id":"convinced.s.01"},{"index":3,"sense_key":"positive%5:00:00:advantageous:00","synset_id":"plus.s.01"},{"index":4,"sense_key":"positive%3:00:04::","synset_id":"positive.a.04"},{"index":5,"sense_key":"positive%5:00:00:formal:01","synset_id":"positive.s.03"},{"index":6,"sense_key":"positive%5:00:00:undeniable:00","synset_id":"incontrovertible.s.01"},{"index":7,"sense_key":"positive%3:01:00::","synset_id":"positivist.a.01"},{"index":8,"sense_key":"positive%3:00:05::","synset_id":"positive.a.08"},{"index":9,"sense_key":"positive%5:00:00:plus:00","synset_id":"positive.s.05"},{"index":10,"sense_key":"positive%5:00:00:charged:00","synset_id":"positive.s.06"},{"index":11,"sense_key":"positive%5:00:00:confident:00","synset_id":"cocksure.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"positive%3:00:05::","call_ids":["senseval3.d001.s045.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"positive%3:00:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":942,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005655,"input_uncached_usd":0.0000471,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.514863791991957}
{"item_id":"senseval3.d001.s045.t021","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"function%1:24:01::","call_ids":["senseval3.d001.s045.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"function%1:24:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":798,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008143000000000001,"input_uncached_usd":0.0000399,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.369267375033814}
{"item_id":"senseval3.d001.s045.t022","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"state%1:14:00::","call_ids":["senseval3.d001.s045.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"state%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":2576,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.001073,"input_uncached_usd":0.0000426,"input_cached_usd":0.0,"output_usd":0.0010304,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.008298375003505}
{"item_id":"senseval3.d001.s046.t001","gold_sense_keys":["federal%5:00:00:national:01"],"candidates":[{"index":1,"sense_key":"federal%5:00:00:national:01","synset_id":"federal.s.01"},{"index":2,"sense_key":"federal%3:01:02::","synset_id":"federal.a.02"},{"index":3,"sense_key":"federal%5:00:00:northern:02","synset_id":"union.s.01"},{"index":4,"sense_key":"federal%3:00:00::","synset_id":"federal.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"federal%5:00:00:national:01","call_ids":["senseval3.d001.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"federal%5:00:00:national:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042085,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.797435124986805}
{"item_id":"senseval3.d001.s046.t004","gold_sense_keys":["level%1:09:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"level%1:09:00::","call_ids":["senseval3.d001.s046.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"level%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007869,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.747457374993246}
{"item_id":"senseval3.d001.s046.t005","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["senseval3.d001.s046.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":993,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.00092645,"input_uncached_usd":0.00004965,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.854572040960193}
{"item_id":"senseval3.d001.s046.t006","gold_sense_keys":["irrational%3:00:00::"],"candidates":[{"index":1,"sense_key":"irrational%3:00:00::","synset_id":"irrational.a.01"},{"index":2,"sense_key":"irrational%3:01:00::","synset_id":"irrational.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"irrational%3:00:00::","call_ids":["senseval3.d001.s046.t006__v1__a1","senseval3.d001.s046.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"irrational%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":985,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00049725,"input_uncached_usd":0.00004925,"input_cached_usd":0.0,"output_usd":0.000448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.2091493330081}
{"item_id":"senseval3.d001.s046.t007","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"behavior%1:04:01::","call_ids":["senseval3.d001.s046.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"behavior%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041885,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.01647412503371}
{"item_id":"senseval3.d001.s046.t009","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["senseval3.d001.s046.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028675,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.467093500017654}
{"item_id":"senseval3.d001.s046.t012","gold_sense_keys":["rational%3:00:00::"],"candidates":[{"index":1,"sense_key":"rational%3:00:00::","synset_id":"rational.a.01"},{"index":2,"sense_key":"rational%5:00:00:mental:00","synset_id":"intellectual.s.01"},{"index":3,"sense_key":"rational%3:01:00::","synset_id":"rational.a.03"},{"index":4,"sense_key":"rational%5:00:00:intellectual:00","synset_id":"rational.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rational%3:00:00::","call_ids":["senseval3.d001.s046.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rational%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031635,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.760348666983191}
{"item_id":"senseval3.d002.s000.t000","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":2,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":3,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":4,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":5,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":6,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":7,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":8,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%3:00:02::","call_ids":["senseval3.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018979999999999998,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8743494580266997}
{"item_id":"senseval3.d002.s000.t002","gold_sense_keys":["night%1:28:00::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"night%1:28:00::","call_ids":["senseval3.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"night%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033529999999999996,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.891981791995931}
{"item_id":"senseval3.d002.s000.t003","gold_sense_keys":["devastate%2:30:00::"],"candidates":[{"index":1,"sense_key":"devastate%2:30:00::","synset_id":"lay_waste_to.v.01"},{"index":2,"sense_key":"devastate%2:30:01::","synset_id":"devastate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"devastate%2:30:00::","call_ids":["senseval3.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"devastate%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":278,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001227,"input_uncached_usd":0.000013899999999999999,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5868619169923477}
{"item_id":"senseval3.d002.s000.t004","gold_sense_keys":["earthquake%1:11:00::"],"candidates":[{"index":1,"sense_key":"earthquake%1:11:00::","synset_id":"earthquake.n.01"},{"index":2,"sense_key":"earthquake%1:26:00::","synset_id":"earthquake.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earthquake%1:11:00::","call_ids":["senseval3.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earthquake%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":269,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053185,"input_uncached_usd":0.00001345,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.818369916989468}
{"item_id":"senseval3.d002.s000.t005","gold_sense_keys":["resident%1:18:00::"],"candidates":[{"index":1,"sense_key":"resident%1:18:00::","synset_id":"resident.n.01"},{"index":2,"sense_key":"resident%1:18:01::","synset_id":"house_physician.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"resident%1:18:00::","call_ids":["senseval3.d002.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"resident%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":289,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017444999999999998,"input_uncached_usd":0.000014449999999999999,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.2787484169821255}
{"item_id":"senseval3.d002.s000.t006","gold_sense_keys":["search%2:35:00::","search%2:39:00::"],"candidates":[{"index":1,"sense_key":"search%2:35:00::","synset_id":"search.v.01"},{"index":2,"sense_key":"search%2:39:00::","synset_id":"search.v.02"},{"index":3,"sense_key":"search%2:31:00::","synset_id":"research.v.02"},{"index":4,"sense_key":"search%2:35:01::","synset_id":"search.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"search%2:39:00::","call_ids":["senseval3.d002.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"search%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046064999999999995,"input_uncached_usd":0.00001905,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.433750791999046}
{"item_id":"senseval3.d002.s000.t007","gold_sense_keys":["comfort%1:04:00::"],"candidates":[{"index":1,"sense_key":"comfort%1:26:00::","synset_id":"comfort.n.01"},{"index":2,"sense_key":"comfort%1:12:00::","synset_id":"comfort.n.02"},{"index":3,"sense_key":"comfort%1:04:00::","synset_id":"consolation.n.02"},{"index":4,"sense_key":"comfort%1:26:01::","synset_id":"ease.n.02"},{"index":5,"sense_key":"comfort%1:26:02::","synset_id":"comfort.n.05"},{"index":6,"sense_key":"comfort%1:06:00::","synset_id":"quilt.n.01"},{"index":7,"sense_key":"comfort%1:04:01::","synset_id":"comfort.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"comfort%1:04:00::","call_ids":["senseval3.d002.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"comfort%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018424999999999998,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.442552541033365}
{"item_id":"senseval3.d002.s000.t008","gold_sense_keys":["solace%1:12:00::","solace%1:26:00::"],"candidates":[{"index":1,"sense_key":"solace%1:12:00::","synset_id":"consolation.n.01"},{"index":2,"sense_key":"solace%1:26:00::","synset_id":"solace.n.02"},{"index":3,"sense_key":"solace%1:04:00::","synset_id":"consolation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"solace%1:26:00::","call_ids":["senseval3.d002.s000.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"solace%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":308,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006874,"input_uncached_usd":0.000015399999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.096248499990907}
{"item_id":"senseval3.d002.s001.t001","gold_sense_keys":["screen%1:06:05::"],"candidates":[{"index":1,"sense_key":"screen%1:06:06::","synset_id":"screen.n.01"},{"index":2,"sense_key":"screen%1:06:04::","synset_id":"blind.n.03"},{"index":3,"sense_key":"screen%1:06:05::","synset_id":"screen.n.03"},{"index":4,"sense_key":"screen%1:06:01::","synset_id":"screen.n.04"},{"index":5,"sense_key":"screen%1:06:00::","synset_id":"screen.n.05"},{"index":6,"sense_key":"screen%1:14:00::","synset_id":"filmdom.n.01"},{"index":7,"sense_key":"screen%1:06:02::","synset_id":"sieve.n.01"},{"index":8,"sense_key":"screen%1:06:07::","synset_id":"screen_door.n.01"},{"index":9,"sense_key":"screen%1:06:03::","synset_id":"screen.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"screen%1:06:05::","call_ids":["senseval3.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"screen%1:06:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001391,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.888556125049945}
{"item_id":"senseval3.d002.s002.t001","gold_sense_keys":["make%2:38:10::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":22,"chosen_sense_key":"make%2:38:00::","call_ids":["senseval3.d002.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":22,"predicted_sense_key":"make%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2212,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0005266,"input_uncached_usd":0.00011059999999999999,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.524859291035682}
{"item_id":"senseval3.d002.s002.t003","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022805,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.125310542003717}
{"item_id":"senseval3.d002.s002.t007","gold_sense_keys":["link%2:35:00::"],"candidates":[{"index":1,"sense_key":"link%2:31:00::","synset_id":"associate.v.01"},{"index":2,"sense_key":"link%2:35:00::","synset_id":"connect.v.01"},{"index":3,"sense_key":"link%2:42:01::","synset_id":"connect.v.03"},{"index":4,"sense_key":"link%2:35:02::","synset_id":"yoke.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"link%2:35:00::","call_ids":["senseval3.d002.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"link%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036214999999999994,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.129660958016757}
{"item_id":"senseval3.d002.s002.t008","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045844999999999995,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.783318958012387}
{"item_id":"senseval3.d002.s003.t000","gold_sense_keys":["vivid%5:00:00:clear:00","vivid%5:00:00:realistic:00"],"candidates":[{"index":1,"sense_key":"vivid%5:00:00:realistic:00","synset_id":"graphic.s.03"},{"index":2,"sense_key":"vivid%5:00:00:clear:00","synset_id":"vivid.s.02"},{"index":3,"sense_key":"vivid%5:00:00:colorful:00","synset_id":"bright.s.01"},{"index":4,"sense_key":"vivid%5:00:00:saturated:03","synset_id":"intense.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vivid%5:00:00:realistic:00","call_ids":["senseval3.d002.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vivid%5:00:00:realistic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036149999999999995,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.400122250022832}
{"item_id":"senseval3.d002.s003.t002","gold_sense_keys":["come%2:38:02::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"come%2:38:02::","call_ids":["senseval3.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"come%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1146,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004988999999999999,"input_uncached_usd":0.0000573,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.291153541998938}
{"item_id":"senseval3.d002.s003.t004","gold_sense_keys":["electronic%3:01:00::"],"candidates":[{"index":1,"sense_key":"electronic%3:01:00::","synset_id":"electronic.a.01"},{"index":2,"sense_key":"electronic%3:01:01::","synset_id":"electronic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"electronic%3:01:00::","call_ids":["senseval3.d002.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"electronic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040775,"input_uncached_usd":0.000017349999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.310827791981865}
{"item_id":"senseval3.d002.s004.t001","gold_sense_keys":["subscriber%1:18:01::"],"candidates":[{"index":1,"sense_key":"subscriber%1:18:02::","synset_id":"subscriber.n.01"},{"index":2,"sense_key":"subscriber%1:18:01::","synset_id":"subscriber.n.02"},{"index":3,"sense_key":"subscriber%1:18:00::","synset_id":"subscriber.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"subscriber%1:18:01::","call_ids":["senseval3.d002.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"subscriber%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023165,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7874922080081888}
{"item_id":"senseval3.d002.s004.t002","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval3.d002.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001929,"input_uncached_usd":0.0000325,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.301855291996617}
{"item_id":"senseval3.d002.s005.t002","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033685,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.071898500027601}
{"item_id":"senseval3.d002.s005.t003","gold_sense_keys":["come_up%2:35:00::"],"candidates":[{"index":1,"sense_key":"come_up%2:36:03::","synset_id":"come_up.v.01"},{"index":2,"sense_key":"come_up%2:30:00::","synset_id":"arise.v.04"},{"index":3,"sense_key":"come_up%2:38:02::","synset_id":"come.v.01"},{"index":4,"sense_key":"come_up%2:38:01::","synset_id":"surface.v.01"},{"index":5,"sense_key":"come_up%2:42:00::","synset_id":"arise.v.02"},{"index":6,"sense_key":"come_up%2:38:03::","synset_id":"rise.v.01"},{"index":7,"sense_key":"come_up%2:42:01::","synset_id":"come_up.v.07"},{"index":8,"sense_key":"come_up%2:35:00::","synset_id":"go_on.v.05"},{"index":9,"sense_key":"come_up%2:40:00::","synset_id":"line_up.v.02"},{"index":10,"sense_key":"come_up%2:38:00::","synset_id":"rise.v.16"},{"index":11,"sense_key":"come_up%2:35:02::","synset_id":"scrape.v.05"},{"index":12,"sense_key":"come_up%2:35:01::","synset_id":"muster.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"come_up%2:35:00::","call_ids":["senseval3.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"come_up%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.000125,"input_uncached_usd":0.0000418,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.6353650420205668}
{"item_id":"senseval3.d002.s005.t004","gold_sense_keys":["teem%2:42:00::"],"candidates":[{"index":1,"sense_key":"teem%2:42:00::","synset_id":"teem.v.01"},{"index":2,"sense_key":"teem%2:38:00::","synset_id":"pour.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"teem%2:42:00::","call_ids":["senseval3.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"teem%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048784999999999996,"input_uncached_usd":0.00002065,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.34685258398531}
{"item_id":"senseval3.d002.s005.t005","gold_sense_keys":["emotional%3:00:00::"],"candidates":[{"index":1,"sense_key":"emotional%3:00:02::","synset_id":"emotional.a.01"},{"index":2,"sense_key":"emotional%3:00:00::","synset_id":"emotional.a.02"},{"index":3,"sense_key":"emotional%3:01:00::","synset_id":"emotional.a.03"},{"index":4,"sense_key":"emotional%5:00:00:agitated:00","synset_id":"aroused.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emotional%3:00:00::","call_ids":["senseval3.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emotional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005674499999999999,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.937600584002212}
{"item_id":"senseval3.d002.s005.t007","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["senseval3.d002.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039644999999999996,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.924562958010938}
{"item_id":"senseval3.d002.s006.t001","gold_sense_keys":["electronic%3:01:00::"],"candidates":[{"index":1,"sense_key":"electronic%3:01:00::","synset_id":"electronic.a.01"},{"index":2,"sense_key":"electronic%3:01:01::","synset_id":"electronic.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"electronic%3:01:00::","call_ids":["senseval3.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"electronic%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043505,"input_uncached_usd":0.00001905,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.389124749985058}
{"item_id":"senseval3.d002.s006.t002","gold_sense_keys":["traffic%1:10:00::"],"candidates":[{"index":1,"sense_key":"traffic%1:14:00::","synset_id":"traffic.n.01"},{"index":2,"sense_key":"traffic%1:04:01::","synset_id":"traffic.n.02"},{"index":3,"sense_key":"traffic%1:10:00::","synset_id":"traffic.n.03"},{"index":4,"sense_key":"traffic%1:04:02::","synset_id":"dealings.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"traffic%1:10:00::","call_ids":["senseval3.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"traffic%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015759999999999998,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9017182079842314}
{"item_id":"senseval3.d002.s007.t000","gold_sense_keys":["time%1:28:03::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"time%1:28:03::","call_ids":["senseval3.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"time%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019544999999999998,"input_uncached_usd":0.00003545,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.013190583034884}
{"item_id":"senseval3.d002.s007.t001","gold_sense_keys":["time%1:28:03::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"time%1:28:03::","call_ids":["senseval3.d002.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"time%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004258,"input_uncached_usd":0.0000354,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.943981124961283}
{"item_id":"senseval3.d002.s007.t004","gold_sense_keys":["subscriber%1:18:01::"],"candidates":[{"index":1,"sense_key":"subscriber%1:18:02::","synset_id":"subscriber.n.01"},{"index":2,"sense_key":"subscriber%1:18:01::","synset_id":"subscriber.n.02"},{"index":3,"sense_key":"subscriber%1:18:00::","synset_id":"subscriber.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"subscriber%1:18:01::","call_ids":["senseval3.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"subscriber%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025749999999999997,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.1209827499696985}
{"item_id":"senseval3.d002.s007.t005","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["senseval3.d002.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003429,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.431603415985592}
{"item_id":"senseval3.d002.s007.t006","gold_sense_keys":["identify%2:32:00::"],"candidates":[{"index":1,"sense_key":"identify%2:31:00::","synset_id":"identify.v.01"},{"index":2,"sense_key":"identify%2:32:00::","synset_id":"name.v.02"},{"index":3,"sense_key":"identify%2:31:04::","synset_id":"identify.v.03"},{"index":4,"sense_key":"identify%2:31:03::","synset_id":"identify.v.04"},{"index":5,"sense_key":"identify%2:31:01::","synset_id":"identify.v.05"},{"index":6,"sense_key":"identify%2:31:02::","synset_id":"identify.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"identify%2:31:00::","call_ids":["senseval3.d002.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"identify%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052085,"input_uncached_usd":0.000028049999999999997,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.797176917025354}
{"item_id":"senseval3.d002.s011.t000","gold_sense_keys":["avenue%1:06:00::"],"candidates":[{"index":1,"sense_key":"avenue%1:04:00::","synset_id":"avenue.n.01"},{"index":2,"sense_key":"avenue%1:06:00::","synset_id":"avenue.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"avenue%1:06:00::","call_ids":["senseval3.d002.s011.t000__v1__a1","senseval3.d002.s011.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"avenue%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":2976,"reasoning_output_tokens":2944},"cost":{"currency":"USD","total_usd":0.00122545,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.0011903999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":23.15244687499944}
{"item_id":"senseval3.d002.s011.t002","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002184,"input_uncached_usd":0.0000328,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.01902304100804}
{"item_id":"senseval3.d002.s011.t003","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"old%3:00:01::","call_ids":["senseval3.d002.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"old%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001916,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8394222910283133}
{"item_id":"senseval3.d002.s011.t004","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020764999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4138144580065273}
{"item_id":"senseval3.d002.s011.t005","gold_sense_keys":["heart%1:08:00::"],"candidates":[{"index":1,"sense_key":"heart%1:09:00::","synset_id":"heart.n.01"},{"index":2,"sense_key":"heart%1:08:00::","synset_id":"heart.n.02"},{"index":3,"sense_key":"heart%1:07:01::","synset_id":"heart.n.03"},{"index":4,"sense_key":"heart%1:15:00::","synset_id":"center.n.01"},{"index":5,"sense_key":"heart%1:09:01::","synset_id":"kernel.n.03"},{"index":6,"sense_key":"heart%1:07:04::","synset_id":"heart.n.06"},{"index":7,"sense_key":"heart%1:25:00::","synset_id":"heart.n.07"},{"index":8,"sense_key":"heart%1:13:00::","synset_id":"heart.n.08"},{"index":9,"sense_key":"heart%1:12:04::","synset_id":"affection.n.01"},{"index":10,"sense_key":"heart%1:06:00::","synset_id":"heart.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"heart%1:08:00::","call_ids":["senseval3.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"heart%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014749999999999998,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.262367874966003}
{"item_id":"senseval3.d002.s012.t000","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":2,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":3,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":4,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":5,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":6,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":7,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":8,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":9,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":10,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":11,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":12,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":13,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":14,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":15,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":16,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":17,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":18,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":19,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":20,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":21,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":23,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":24,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":25,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":26,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":27,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":28,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":29,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":30,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":31,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":32,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"fall%2:38:15::","call_ids":["senseval3.d002.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"fall%2:38:15::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1514,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006453000000000001,"input_uncached_usd":0.0000757,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.728027625009418}
{"item_id":"senseval3.d002.s012.t001","gold_sense_keys":["poster%1:10:00::"],"candidates":[{"index":1,"sense_key":"poster%1:10:00::","synset_id":"poster.n.01"},{"index":2,"sense_key":"poster%1:18:00::","synset_id":"bill_poster.n.01"},{"index":3,"sense_key":"poster%1:05:00::","synset_id":"post_horse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poster%1:10:00::","call_ids":["senseval3.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poster%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":591,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025545,"input_uncached_usd":0.00001905,"input_cached_usd":0.0,"output_usd":0.0002364,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3872712919837795}
{"item_id":"senseval3.d002.s012.t002","gold_sense_keys":["frame%1:06:00::"],"candidates":[{"index":1,"sense_key":"frame%1:06:02::","synset_id":"frame.n.01"},{"index":2,"sense_key":"frame%1:06:03::","synset_id":"frame.n.02"},{"index":3,"sense_key":"frame%1:08:01::","synset_id":"human_body.n.01"},{"index":4,"sense_key":"frame%1:28:00::","synset_id":"inning.n.01"},{"index":5,"sense_key":"frame%1:10:00::","synset_id":"frame.n.05"},{"index":6,"sense_key":"frame%1:10:01::","synset_id":"frame.n.06"},{"index":7,"sense_key":"frame%1:09:00::","synset_id":"frame_of_reference.n.02"},{"index":8,"sense_key":"frame%1:08:00::","synset_id":"skeletal_system.n.01"},{"index":9,"sense_key":"frame%1:06:01::","synset_id":"skeleton.n.04"},{"index":10,"sense_key":"frame%1:06:00::","synset_id":"frame.n.10"},{"index":11,"sense_key":"frame%1:04:01::","synset_id":"frame.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"frame%1:06:01::","call_ids":["senseval3.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"frame%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":2640,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.00109035,"input_uncached_usd":0.00003435,"input_cached_usd":0.0,"output_usd":0.0010559999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.628599707968533}
{"item_id":"senseval3.d002.s012.t003","gold_sense_keys":["glass%1:27:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"glass%1:27:00::","call_ids":["senseval3.d002.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"glass%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028724999999999996,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.9695554170175456}
{"item_id":"senseval3.d002.s012.t004","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":3600,"reasoning_output_tokens":3584},"cost":{"currency":"USD","total_usd":0.0014726,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.0014399999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":22.854388833977282}
{"item_id":"senseval3.d002.s012.t006","gold_sense_keys":["open%3:00:01::"],"candidates":[{"index":1,"sense_key":"open%3:00:01::","synset_id":"open.a.01"},{"index":2,"sense_key":"open%3:00:02::","synset_id":"open.a.02"},{"index":3,"sense_key":"open%5:00:00:unprotected:00","synset_id":"exposed.s.01"},{"index":4,"sense_key":"open%5:00:00:public:00","synset_id":"open.s.02"},{"index":5,"sense_key":"open%3:00:08::","synset_id":"open.a.05"},{"index":6,"sense_key":"open%5:00:00:available:00","synset_id":"open.s.03"},{"index":7,"sense_key":"open%5:00:00:unrestricted:00","synset_id":"open.s.04"},{"index":8,"sense_key":"open%5:00:00:vulnerable:00","synset_id":"assailable.s.01"},{"index":9,"sense_key":"open%5:00:00:coarse:00","synset_id":"loose.s.06"},{"index":10,"sense_key":"open%5:00:00:unenclosed:00","synset_id":"open.s.07"},{"index":11,"sense_key":"open%3:00:04::","synset_id":"open.a.11"},{"index":12,"sense_key":"open%5:00:00:unsettled:02","synset_id":"open.s.08"},{"index":13,"sense_key":"open%5:00:00:unsealed:01","synset_id":"open.s.09"},{"index":14,"sense_key":"open%5:00:00:unconstricted:00","synset_id":"open.s.10"},{"index":15,"sense_key":"open%3:00:00::","synset_id":"receptive.a.02"},{"index":16,"sense_key":"open%3:00:07::","synset_id":"overt.a.01"},{"index":17,"sense_key":"open%5:00:00:nonunion:00","synset_id":"open.s.11"},{"index":18,"sense_key":"open%5:00:00:susceptible:00","synset_id":"capable.s.01"},{"index":19,"sense_key":"open%5:00:00:unobstructed:00","synset_id":"clear.s.02"},{"index":20,"sense_key":"open%5:00:00:ingenuous:00","synset_id":"candid.s.03"},{"index":21,"sense_key":"open%5:00:00:active:06","synset_id":"open.s.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"open%3:00:01::","call_ids":["senseval3.d002.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"open%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1099,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00039414999999999996,"input_uncached_usd":0.00005495,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.076336917001754}
{"item_id":"senseval3.d002.s012.t007","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001926,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9896647079731338}
{"item_id":"senseval3.d002.s015.t000","gold_sense_keys":["favorite%5:00:00:loved:00"],"candidates":[{"index":1,"sense_key":"favorite%5:00:00:popular:00","synset_id":"favorite.s.01"},{"index":2,"sense_key":"favorite%5:00:00:loved:00","synset_id":"favored.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favorite%5:00:00:loved:00","call_ids":["senseval3.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favorite%5:00:00:loved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005114,"input_uncached_usd":0.0000182,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.67465658299625}
{"item_id":"senseval3.d002.s015.t001","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041305,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.752703415986616}
{"item_id":"senseval3.d002.s015.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["senseval3.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005278500000000001,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.196422207984142}
{"item_id":"senseval3.d002.s015.t003","gold_sense_keys":["start%2:42:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval3.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046114999999999996,"input_uncached_usd":0.00004515,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.129951667040586}
{"item_id":"senseval3.d002.s016.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020169999999999997,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3070337920216843}
{"item_id":"senseval3.d002.s016.t002","gold_sense_keys":["begin%2:42:00::","begin%2:42:04::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"begin%2:30:01::","call_ids":["senseval3.d002.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"begin%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":3088,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.0012750499999999998,"input_uncached_usd":0.00003985,"input_cached_usd":0.0,"output_usd":0.0012351999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.538466250000056}
{"item_id":"senseval3.d002.s016.t003","gold_sense_keys":["glance%2:39:00::"],"candidates":[{"index":1,"sense_key":"glance%2:39:00::","synset_id":"glance.v.01"},{"index":2,"sense_key":"glance%2:38:00::","synset_id":"glance.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"glance%2:39:00::","call_ids":["senseval3.d002.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"glance%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045994999999999993,"input_uncached_usd":0.00001835,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.137730249960441}
{"item_id":"senseval3.d002.s016.t004","gold_sense_keys":["table%1:14:01::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"table%1:06:01::","call_ids":["senseval3.d002.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"table%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001613,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3698300420073792}
{"item_id":"senseval3.d002.s016.t005","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval3.d002.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007879999999999999,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9297634160029702}
{"item_id":"senseval3.d002.s016.t007","gold_sense_keys":["guilty%5:00:00:ashamed:00"],"candidates":[{"index":1,"sense_key":"guilty%3:00:00::","synset_id":"guilty.a.01"},{"index":2,"sense_key":"guilty%5:00:00:ashamed:00","synset_id":"guilty.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"guilty%5:00:00:ashamed:00","call_ids":["senseval3.d002.s016.t007__v1__a1","senseval3.d002.s016.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"guilty%5:00:00:ashamed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":839,"cached_input_tokens":0,"output_tokens":2912,"reasoning_output_tokens":2880},"cost":{"currency":"USD","total_usd":0.00120675,"input_uncached_usd":0.00004195,"input_cached_usd":0.0,"output_usd":0.0011648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.026360041927546}
{"item_id":"senseval3.d002.s016.t009","gold_sense_keys":["mouth%2:32:01::"],"candidates":[{"index":1,"sense_key":"mouth%2:32:00::","synset_id":"talk.v.02"},{"index":2,"sense_key":"mouth%2:32:01::","synset_id":"mouth.v.02"},{"index":3,"sense_key":"mouth%2:35:00::","synset_id":"mouth.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mouth%2:32:01::","call_ids":["senseval3.d002.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mouth%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005128000000000001,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.457762042002287}
{"item_id":"senseval3.d002.s016.t010","gold_sense_keys":["word%1:10:00::"],"candidates":[{"index":1,"sense_key":"word%1:10:00::","synset_id":"word.n.01"},{"index":2,"sense_key":"word%1:10:03::","synset_id":"word.n.02"},{"index":3,"sense_key":"word%1:10:02::","synset_id":"news.n.01"},{"index":4,"sense_key":"word%1:10:06::","synset_id":"word.n.04"},{"index":5,"sense_key":"word%1:10:04::","synset_id":"discussion.n.02"},{"index":6,"sense_key":"word%1:10:01::","synset_id":"parole.n.01"},{"index":7,"sense_key":"word%1:23:00::","synset_id":"word.n.07"},{"index":8,"sense_key":"word%1:18:00::","synset_id":"son.n.02"},{"index":9,"sense_key":"word%1:10:05::","synset_id":"password.n.01"},{"index":10,"sense_key":"word%1:10:07::","synset_id":"bible.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"word%1:10:00::","call_ids":["senseval3.d002.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"word%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024625,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.544596375024412}
{"item_id":"senseval3.d002.s016.t011","gold_sense_keys":["earthquake%1:11:00::"],"candidates":[{"index":1,"sense_key":"earthquake%1:11:00::","synset_id":"earthquake.n.01"},{"index":2,"sense_key":"earthquake%1:26:00::","synset_id":"earthquake.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earthquake%1:11:00::","call_ids":["senseval3.d002.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earthquake%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030680000000000003,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.300817666982766}
{"item_id":"senseval3.d002.s017.t000","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":2,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":3,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":4,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":5,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":6,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":7,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":8,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%3:00:02::","call_ids":["senseval3.d002.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024605,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.82958033302566}
{"item_id":"senseval3.d002.s017.t001","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d002.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2146,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00034449999999999997,"input_uncached_usd":0.00010729999999999999,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.876287417020649}
{"item_id":"senseval3.d002.s017.t003","gold_sense_keys":["pass%2:30:01::","pass%2:38:03::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"pass%2:30:01::","call_ids":["senseval3.d002.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"pass%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1421,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00035904999999999997,"input_uncached_usd":0.00007104999999999999,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.084073083009571}
{"item_id":"senseval3.d002.s018.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["senseval3.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003212,"input_uncached_usd":0.0000332,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.162862750003114}
{"item_id":"senseval3.d002.s018.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1893,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00028025,"input_uncached_usd":0.00009465,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.13758300000336}
{"item_id":"senseval3.d002.s018.t002","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010539999999999999,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9961779579753056}
{"item_id":"senseval3.d002.s018.t003","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval3.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00081875,"input_uncached_usd":0.00004435,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.96200779202627}
{"item_id":"senseval3.d002.s018.t004","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shake%2:38:01::","call_ids":["senseval3.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shake%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032065,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.069314249965828}
{"item_id":"senseval3.d002.s018.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["senseval3.d002.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043775,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.9794579580193385}
{"item_id":"senseval3.d002.s018.t006","gold_sense_keys":["toy%1:06:00::"],"candidates":[{"index":1,"sense_key":"toy%1:06:00::","synset_id":"plaything.n.01"},{"index":2,"sense_key":"toy%1:06:02::","synset_id":"toy.n.02"},{"index":3,"sense_key":"toy%1:06:03::","synset_id":"toy.n.03"},{"index":4,"sense_key":"toy%1:06:01::","synset_id":"miniature.n.02"},{"index":5,"sense_key":"toy%1:05:00::","synset_id":"toy_dog.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"toy%1:06:00::","call_ids":["senseval3.d002.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"toy%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013005,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.488729709002655}
{"item_id":"senseval3.d002.s018.t007","gold_sense_keys":["block%1:06:00::"],"candidates":[{"index":1,"sense_key":"block%1:06:00::","synset_id":"block.n.01"},{"index":2,"sense_key":"block%1:15:00::","synset_id":"block.n.02"},{"index":3,"sense_key":"block%1:25:01::","synset_id":"block.n.03"},{"index":4,"sense_key":"block%1:14:00::","synset_id":"block.n.04"},{"index":5,"sense_key":"block%1:06:06::","synset_id":"block.n.05"},{"index":6,"sense_key":"block%1:23:00::","synset_id":"block.n.06"},{"index":7,"sense_key":"block%1:09:00::","synset_id":"block.n.07"},{"index":8,"sense_key":"block%1:06:02::","synset_id":"pulley.n.01"},{"index":9,"sense_key":"block%1:06:04::","synset_id":"engine_block.n.01"},{"index":10,"sense_key":"block%1:06:01::","synset_id":"blockage.n.02"},{"index":11,"sense_key":"block%1:06:05::","synset_id":"auction_block.n.01"},{"index":12,"sense_key":"block%1:04:00::","synset_id":"blocking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"block%1:25:01::","call_ids":["senseval3.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"block%1:25:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002518,"input_uncached_usd":0.0000406,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9652342920308}
{"item_id":"senseval3.d002.s018.t008","gold_sense_keys":["toss%2:35:05::"],"candidates":[{"index":1,"sense_key":"toss%2:35:04::","synset_id":"flip.v.06"},{"index":2,"sense_key":"toss%2:38:00::","synset_id":"flip.v.01"},{"index":3,"sense_key":"toss%2:35:05::","synset_id":"chuck.v.01"},{"index":4,"sense_key":"toss%2:38:01::","synset_id":"convulse.v.03"},{"index":5,"sense_key":"toss%2:40:00::","synset_id":"discard.v.01"},{"index":6,"sense_key":"toss%2:35:00::","synset_id":"toss.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"toss%2:38:01::","call_ids":["senseval3.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"toss%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004927,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.883412000024691}
{"item_id":"senseval3.d002.s021.t000","gold_sense_keys":["library%1:06:00::","library%1:06:01::","library%1:06:02::"],"candidates":[{"index":1,"sense_key":"library%1:06:01::","synset_id":"library.n.01"},{"index":2,"sense_key":"library%1:14:00::","synset_id":"library.n.02"},{"index":3,"sense_key":"library%1:06:00::","synset_id":"library.n.03"},{"index":4,"sense_key":"library%1:14:01::","synset_id":"library.n.04"},{"index":5,"sense_key":"library%1:06:02::","synset_id":"library.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"library%1:06:02::","call_ids":["senseval3.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"library%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002073,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6902635829756036}
{"item_id":"senseval3.d002.s021.t001","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"hit%2:35:01::","call_ids":["senseval3.d002.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"hit%2:35:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1002,"cached_input_tokens":0,"output_tokens":2833,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.0011833,"input_uncached_usd":0.0000501,"input_cached_usd":0.0,"output_usd":0.0011332,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.861389832978602}
{"item_id":"senseval3.d002.s022.t000","gold_sense_keys":["endless%5:00:00:long:02"],"candidates":[{"index":1,"sense_key":"endless%5:00:00:long:02","synset_id":"endless.s.01"},{"index":2,"sense_key":"endless%5:00:02:infinite:00","synset_id":"endless.s.02"},{"index":3,"sense_key":"endless%5:00:00:infinite:00","synset_id":"dateless.s.01"},{"index":4,"sense_key":"endless%5:00:00:continuous:01","synset_id":"endless.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"endless%5:00:00:long:02","call_ids":["senseval3.d002.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"endless%5:00:00:long:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023155,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7513413749984466}
{"item_id":"senseval3.d002.s022.t001","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034625,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.015220666013192}
{"item_id":"senseval3.d002.s022.t002","gold_sense_keys":["wonder%2:32:01::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wonder%2:32:01::","call_ids":["senseval3.d002.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wonder%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002807,"input_uncached_usd":0.000018299999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.546965625020675}
{"item_id":"senseval3.d002.s022.t004","gold_sense_keys":["window%1:06:00::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"window%1:06:00::","call_ids":["senseval3.d002.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"window%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039315,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.782206457981374}
{"item_id":"senseval3.d002.s022.t005","gold_sense_keys":["buckle%2:30:00::","buckle%2:35:01::"],"candidates":[{"index":1,"sense_key":"buckle%2:35:00::","synset_id":"buckle.v.01"},{"index":2,"sense_key":"buckle%2:35:01::","synset_id":"buckle.v.02"},{"index":3,"sense_key":"buckle%2:30:00::","synset_id":"heave.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"buckle%2:30:00::","call_ids":["senseval3.d002.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"buckle%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035654999999999997,"input_uncached_usd":0.000017349999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.630362208990846}
{"item_id":"senseval3.d002.s022.t006","gold_sense_keys":["shower%2:35:00::"],"candidates":[{"index":1,"sense_key":"shower%2:40:00::","synset_id":"lavish.v.01"},{"index":2,"sense_key":"shower%2:35:00::","synset_id":"shower.v.02"},{"index":3,"sense_key":"shower%2:29:00::","synset_id":"shower.v.03"},{"index":4,"sense_key":"shower%2:43:00::","synset_id":"shower.v.04"},{"index":5,"sense_key":"shower%2:34:12::","synset_id":"shower.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"shower%2:43:00::","call_ids":["senseval3.d002.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"shower%2:43:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010429999999999999,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2298852920066565}
{"item_id":"senseval3.d002.s022.t007","gold_sense_keys":["glass%1:27:00::"],"candidates":[{"index":1,"sense_key":"glass%1:27:00::","synset_id":"glass.n.01"},{"index":2,"sense_key":"glass%1:06:00::","synset_id":"glass.n.02"},{"index":3,"sense_key":"glass%1:23:00::","synset_id":"glass.n.03"},{"index":4,"sense_key":"glass%1:06:03::","synset_id":"field_glass.n.01"},{"index":5,"sense_key":"glass%1:06:01::","synset_id":"methamphetamine.n.01"},{"index":6,"sense_key":"glass%1:06:02::","synset_id":"looking_glass.n.01"},{"index":7,"sense_key":"glass%1:06:04::","synset_id":"glass.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"glass%1:27:00::","call_ids":["senseval3.d002.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"glass%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018395,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5498367909458466}
{"item_id":"senseval3.d002.s023.t001","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001962,"input_uncached_usd":0.0000362,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.174126416037325}
{"item_id":"senseval3.d002.s023.t002","gold_sense_keys":["fall%2:38:03::"],"candidates":[{"index":1,"sense_key":"fall%2:38:03::","synset_id":"fall.v.01"},{"index":2,"sense_key":"fall%2:38:00::","synset_id":"descend.v.01"},{"index":3,"sense_key":"fall%2:30:08::","synset_id":"fall.v.03"},{"index":4,"sense_key":"fall%2:42:03::","synset_id":"fall.v.04"},{"index":5,"sense_key":"fall%2:43:00::","synset_id":"precipitate.v.03"},{"index":6,"sense_key":"fall%2:41:02::","synset_id":"fall.v.06"},{"index":7,"sense_key":"fall%2:42:00::","synset_id":"fall.v.07"},{"index":8,"sense_key":"fall%2:42:04::","synset_id":"fall.v.08"},{"index":9,"sense_key":"fall%2:40:05::","synset_id":"fall.v.09"},{"index":10,"sense_key":"fall%2:30:00::","synset_id":"fall.v.10"},{"index":11,"sense_key":"fall%2:30:06::","synset_id":"decrease.v.01"},{"index":12,"sense_key":"fall%2:41:01::","synset_id":"fall.v.12"},{"index":13,"sense_key":"fall%2:41:00::","synset_id":"fall.v.13"},{"index":14,"sense_key":"fall%2:40:03::","synset_id":"fall.v.14"},{"index":15,"sense_key":"fall%2:38:01::","synset_id":"fall.v.15"},{"index":16,"sense_key":"fall%2:42:02::","synset_id":"fall.v.16"},{"index":17,"sense_key":"fall%2:41:15::","synset_id":"fall.v.17"},{"index":18,"sense_key":"fall%2:40:12::","synset_id":"fall.v.18"},{"index":19,"sense_key":"fall%2:40:09::","synset_id":"accrue.v.02"},{"index":20,"sense_key":"fall%2:40:08::","synset_id":"fall.v.20"},{"index":21,"sense_key":"fall%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"fall%2:38:11::","synset_id":"fall.v.22"},{"index":23,"sense_key":"fall%2:38:15::","synset_id":"fall.v.23"},{"index":24,"sense_key":"fall%2:38:14::","synset_id":"fall.v.24"},{"index":25,"sense_key":"fall%2:35:00::","synset_id":"hang.v.05"},{"index":26,"sense_key":"fall%2:30:11::","synset_id":"fall.v.26"},{"index":27,"sense_key":"fall%2:30:14::","synset_id":"fall.v.27"},{"index":28,"sense_key":"fall%2:30:10::","synset_id":"fall.v.28"},{"index":29,"sense_key":"fall%2:30:09::","synset_id":"fall.v.29"},{"index":30,"sense_key":"fall%2:30:12::","synset_id":"fall.v.30"},{"index":31,"sense_key":"fall%2:30:15::","synset_id":"fall.v.31"},{"index":32,"sense_key":"fall%2:30:13::","synset_id":"fall.v.32"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":23,"chosen_sense_key":"fall%2:38:15::","call_ids":["senseval3.d002.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":23,"predicted_sense_key":"fall%2:38:15::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1502,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00044029999999999997,"input_uncached_usd":0.0000751,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.7832835000008345}
{"item_id":"senseval3.d002.s024.t001","gold_sense_keys":["paint%1:06:00::"],"candidates":[{"index":1,"sense_key":"paint%1:06:00::","synset_id":"paint.n.01"},{"index":2,"sense_key":"paint%1:15:00::","synset_id":"key.n.09"},{"index":3,"sense_key":"paint%1:06:01::","synset_id":"rouge.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"paint%1:06:00::","call_ids":["senseval3.d002.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"paint%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010309999999999999,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0885242499643937}
{"item_id":"senseval3.d002.s024.t002","gold_sense_keys":["shop%1:06:01::"],"candidates":[{"index":1,"sense_key":"shop%1:06:00::","synset_id":"shop.n.01"},{"index":2,"sense_key":"shop%1:06:01::","synset_id":"workshop.n.01"},{"index":3,"sense_key":"shop%1:04:00::","synset_id":"shop_class.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shop%1:06:01::","call_ids":["senseval3.d002.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shop%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":340,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002282,"input_uncached_usd":0.000017,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.302844749996439}
{"item_id":"senseval3.d002.s024.t003","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021614999999999997,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2291549170040525}
{"item_id":"senseval3.d002.s024.t004","gold_sense_keys":["send%2:38:01::"],"candidates":[{"index":1,"sense_key":"send%2:38:01::","synset_id":"send.v.01"},{"index":2,"sense_key":"send%2:35:00::","synset_id":"send.v.02"},{"index":3,"sense_key":"send%2:32:00::","synset_id":"mail.v.02"},{"index":4,"sense_key":"send%2:38:00::","synset_id":"transport.v.04"},{"index":5,"sense_key":"send%2:33:00::","synset_id":"station.v.01"},{"index":6,"sense_key":"send%2:32:02::","synset_id":"send.v.06"},{"index":7,"sense_key":"send%2:40:00::","synset_id":"commit.v.03"},{"index":8,"sense_key":"send%2:32:03::","synset_id":"air.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"send%2:38:01::","call_ids":["senseval3.d002.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"send%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002393,"input_uncached_usd":0.0000281,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6861730830278248}
{"item_id":"senseval3.d002.s024.t006","gold_sense_keys":["cloud%1:19:01::"],"candidates":[{"index":1,"sense_key":"cloud%1:19:01::","synset_id":"cloud.n.01"},{"index":2,"sense_key":"cloud%1:17:00::","synset_id":"cloud.n.02"},{"index":3,"sense_key":"cloud%1:26:01::","synset_id":"cloud.n.03"},{"index":4,"sense_key":"cloud%1:26:02::","synset_id":"cloud.n.04"},{"index":5,"sense_key":"cloud%1:26:00::","synset_id":"cloud.n.05"},{"index":6,"sense_key":"cloud%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cloud%1:19:01::","call_ids":["senseval3.d002.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cloud%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020724999999999997,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7067223329795524}
{"item_id":"senseval3.d002.s024.t007","gold_sense_keys":["black%3:00:01::"],"candidates":[{"index":1,"sense_key":"black%3:00:01::","synset_id":"black.a.01"},{"index":2,"sense_key":"black%3:00:02::","synset_id":"black.a.02"},{"index":3,"sense_key":"black%5:00:00:angry:00","synset_id":"black.s.01"},{"index":4,"sense_key":"black%5:00:00:hopeless:00","synset_id":"black.s.02"},{"index":5,"sense_key":"black%5:00:00:evil:00","synset_id":"black.s.03"},{"index":6,"sense_key":"black%5:00:00:unfortunate:00","synset_id":"black.s.04"},{"index":7,"sense_key":"black%5:00:00:colored:00","synset_id":"black.s.05"},{"index":8,"sense_key":"black%5:00:00:dark:01","synset_id":"black.s.06"},{"index":9,"sense_key":"black%5:00:00:sarcastic:00","synset_id":"black.s.07"},{"index":10,"sense_key":"black%5:00:00:covert:00","synset_id":"black.s.08"},{"index":11,"sense_key":"black%5:00:00:illegal:00","synset_id":"bootleg.s.01"},{"index":12,"sense_key":"black%5:00:00:dishonorable:00","synset_id":"black.s.10"},{"index":13,"sense_key":"black%5:00:00:undiluted:00","synset_id":"black.s.11"},{"index":14,"sense_key":"black%5:00:00:dirty:01","synset_id":"black.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"black%3:00:01::","call_ids":["senseval3.d002.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"black%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":932,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00018099999999999998,"input_uncached_usd":0.0000466,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.782511958037503}
{"item_id":"senseval3.d002.s024.t008","gold_sense_keys":["smoke%1:19:00::","smoke%1:22:00::"],"candidates":[{"index":1,"sense_key":"smoke%1:19:00::","synset_id":"smoke.n.01"},{"index":2,"sense_key":"smoke%1:22:00::","synset_id":"smoke.n.02"},{"index":3,"sense_key":"smoke%1:10:00::","synset_id":"smoke.n.03"},{"index":4,"sense_key":"smoke%1:07:00::","synset_id":"smoke.n.04"},{"index":5,"sense_key":"smoke%1:06:00::","synset_id":"roll_of_tobacco.n.01"},{"index":6,"sense_key":"smoke%1:06:01::","synset_id":"pot.n.09"},{"index":7,"sense_key":"smoke%1:04:00::","synset_id":"smoke.n.07"},{"index":8,"sense_key":"smoke%1:04:02::","synset_id":"fastball.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"smoke%1:22:00::","call_ids":["senseval3.d002.s024.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"smoke%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004950999999999999,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.53988895798102}
{"item_id":"senseval3.d002.s024.t009","gold_sense_keys":["air%1:15:00::","air%1:27:00::"],"candidates":[{"index":1,"sense_key":"air%1:27:00::","synset_id":"air.n.01"},{"index":2,"sense_key":"air%1:15:00::","synset_id":"air.n.02"},{"index":3,"sense_key":"air%1:07:00::","synset_id":"air.n.03"},{"index":4,"sense_key":"air%1:19:00::","synset_id":"breeze.n.01"},{"index":5,"sense_key":"air%1:15:01::","synset_id":"atmosphere.n.03"},{"index":6,"sense_key":"air%1:27:01::","synset_id":"air.n.06"},{"index":7,"sense_key":"air%1:10:01::","synset_id":"tune.n.01"},{"index":8,"sense_key":"air%1:10:02::","synset_id":"air.n.08"},{"index":9,"sense_key":"air%1:04:00::","synset_id":"air_travel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"air%1:15:00::","call_ids":["senseval3.d002.s024.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"air%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024324999999999998,"input_uncached_usd":0.00003205,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.2959654169972055}
{"item_id":"senseval3.d002.s027.t001","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001581,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2718298750114627}
{"item_id":"senseval3.d002.s028.t000","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028314999999999997,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.968704583996441}
{"item_id":"senseval3.d002.s028.t001","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"shake%2:38:01::","call_ids":["senseval3.d002.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"shake%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002936,"input_uncached_usd":0.0000312,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.557391083973926}
{"item_id":"senseval3.d002.s028.t002","gold_sense_keys":["like_hell%4:02:00::"],"candidates":[{"index":1,"sense_key":"like_hell%4:02:00::","synset_id":"like_hell.r.01"},{"index":2,"sense_key":"like_hell%4:02:01::","synset_id":"like_hell.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"like_hell%4:02:00::","call_ids":["senseval3.d002.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"like_hell%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003815,"input_uncached_usd":0.0000167,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.646173333982006}
{"item_id":"senseval3.d002.s028.t003","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d002.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1163,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00021815,"input_uncached_usd":0.00005815,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2493145409971476}
{"item_id":"senseval3.d002.s028.t004","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1864,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002788,"input_uncached_usd":0.0000932,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.934818333014846}
{"item_id":"senseval3.d002.s029.t001","gold_sense_keys":["explode%2:30:04::"],"candidates":[{"index":1,"sense_key":"explode%2:30:01::","synset_id":"explode.v.01"},{"index":2,"sense_key":"explode%2:30:00::","synset_id":"explode.v.02"},{"index":3,"sense_key":"explode%2:31:00::","synset_id":"explode.v.03"},{"index":4,"sense_key":"explode%2:30:03::","synset_id":"explode.v.04"},{"index":5,"sense_key":"explode%2:35:00::","synset_id":"explode.v.05"},{"index":6,"sense_key":"explode%2:32:02::","synset_id":"explode.v.06"},{"index":7,"sense_key":"explode%2:32:00::","synset_id":"explode.v.07"},{"index":8,"sense_key":"explode%2:31:01::","synset_id":"explode.v.08"},{"index":9,"sense_key":"explode%2:30:04::","synset_id":"detonate.v.02"},{"index":10,"sense_key":"explode%2:30:02::","synset_id":"explode.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"explode%2:30:04::","call_ids":["senseval3.d002.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"explode%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00085625,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.884695584012661}
{"item_id":"senseval3.d002.s029.t002","gold_sense_keys":["burn%2:30:00::"],"candidates":[{"index":1,"sense_key":"burn%2:30:02::","synset_id":"burn.v.01"},{"index":2,"sense_key":"burn%2:43:01::","synset_id":"burn.v.02"},{"index":3,"sense_key":"burn%2:30:00::","synset_id":"burn.v.03"},{"index":4,"sense_key":"burn%2:39:00::","synset_id":"bite.v.02"},{"index":5,"sense_key":"burn%2:43:02::","synset_id":"burn.v.05"},{"index":6,"sense_key":"burn%2:37:00::","synset_id":"burn.v.06"},{"index":7,"sense_key":"burn%2:30:01::","synset_id":"burn.v.07"},{"index":8,"sense_key":"burn%2:41:00::","synset_id":"burn.v.08"},{"index":9,"sense_key":"burn%2:40:01::","synset_id":"burn.v.09"},{"index":10,"sense_key":"burn%2:39:04::","synset_id":"burn.v.10"},{"index":11,"sense_key":"burn%2:30:08::","synset_id":"cauterize.v.01"},{"index":12,"sense_key":"burn%2:29:00::","synset_id":"sunburn.v.01"},{"index":13,"sense_key":"burn%2:36:05::","synset_id":"cut.v.21"},{"index":14,"sense_key":"burn%2:34:00::","synset_id":"burn_off.v.01"},{"index":15,"sense_key":"burn%2:30:03::","synset_id":"burn.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"burn%2:30:00::","call_ids":["senseval3.d002.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"burn%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00053295,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.779175500036217}
{"item_id":"senseval3.d002.s029.t004","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"thing%1:09:02::","call_ids":["senseval3.d002.s029.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"thing%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007545,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.50198908400489}
{"item_id":"senseval3.d002.s029.t005","gold_sense_keys":["peaceful%3:00:00::"],"candidates":[{"index":1,"sense_key":"peaceful%3:00:00::","synset_id":"peaceful.a.01"},{"index":2,"sense_key":"peaceful%5:00:00:nonviolent:00","synset_id":"passive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"peaceful%3:00:00::","call_ids":["senseval3.d002.s029.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"peaceful%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":309,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00027785,"input_uncached_usd":0.00001545,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.6102431250037625}
{"item_id":"senseval3.d002.s030.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["senseval3.d002.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018759999999999998,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.379937459016219}
{"item_id":"senseval3.d002.s030.t001","gold_sense_keys":["car%1:06:00::"],"candidates":[{"index":1,"sense_key":"car%1:06:00::","synset_id":"car.n.01"},{"index":2,"sense_key":"car%1:06:01::","synset_id":"car.n.02"},{"index":3,"sense_key":"car%1:06:03::","synset_id":"car.n.03"},{"index":4,"sense_key":"car%1:06:02::","synset_id":"car.n.04"},{"index":5,"sense_key":"car%1:06:04::","synset_id":"cable_car.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"car%1:06:00::","call_ids":["senseval3.d002.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"car%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020769999999999998,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.415210790990386}
{"item_id":"senseval3.d002.s030.t002","gold_sense_keys":["alarm%1:06:00::"],"candidates":[{"index":1,"sense_key":"alarm%1:12:00::","synset_id":"alarm.n.01"},{"index":2,"sense_key":"alarm%1:06:00::","synset_id":"alarm.n.02"},{"index":3,"sense_key":"alarm%1:10:00::","synset_id":"alarm.n.03"},{"index":4,"sense_key":"alarm%1:06:01::","synset_id":"alarm_clock.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"alarm%1:10:00::","call_ids":["senseval3.d002.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"alarm%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.000049749999999999996,"input_uncached_usd":0.000017749999999999998,"input_cached_usd":0.0,"output_usd":0.000032,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.2507615839713253}
{"item_id":"senseval3.d002.s030.t003","gold_sense_keys":["go_off%2:30:00::"],"candidates":[{"index":1,"sense_key":"go_off%2:38:00::","synset_id":"abscond.v.01"},{"index":2,"sense_key":"go_off%2:30:00::","synset_id":"go_off.v.02"},{"index":3,"sense_key":"go_off%2:33:00::","synset_id":"fire.v.05"},{"index":4,"sense_key":"go_off%2:35:00::","synset_id":"go_off.v.04"},{"index":5,"sense_key":"go_off%2:30:01::","synset_id":"go_off.v.05"},{"index":6,"sense_key":"go_off%2:30:02::","synset_id":"implode.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"go_off%2:30:00::","call_ids":["senseval3.d002.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"go_off%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049015,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.4662504169973545}
{"item_id":"senseval3.d002.s031.t000","gold_sense_keys":["cat%1:05:00::"],"candidates":[{"index":1,"sense_key":"cat%1:05:00::","synset_id":"cat.n.01"},{"index":2,"sense_key":"cat%1:18:01::","synset_id":"guy.n.01"},{"index":3,"sense_key":"cat%1:18:00::","synset_id":"cat.n.03"},{"index":4,"sense_key":"cat%1:06:02::","synset_id":"kat.n.01"},{"index":5,"sense_key":"cat%1:06:00::","synset_id":"cat-o'-nine-tails.n.01"},{"index":6,"sense_key":"cat%1:06:01::","synset_id":"caterpillar.n.02"},{"index":7,"sense_key":"cat%1:05:02::","synset_id":"big_cat.n.01"},{"index":8,"sense_key":"cat%1:04:00::","synset_id":"computerized_tomography.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cat%1:05:00::","call_ids":["senseval3.d002.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cat%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021549999999999998,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.541251040995121}
{"item_id":"senseval3.d002.s031.t001","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018355,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2294237919850275}
{"item_id":"senseval3.d002.s031.t002","gold_sense_keys":["nervous%5:00:00:troubled:00"],"candidates":[{"index":1,"sense_key":"nervous%5:00:00:tense:03","synset_id":"nervous.s.01"},{"index":2,"sense_key":"nervous%5:00:00:troubled:00","synset_id":"anxious.s.02"},{"index":3,"sense_key":"nervous%3:01:00::","synset_id":"nervous.a.03"},{"index":4,"sense_key":"nervous%5:00:00:excited:00","synset_id":"aflutter.s.01"},{"index":5,"sense_key":"nervous%5:00:00:excitable:00","synset_id":"skittish.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nervous%5:00:00:troubled:00","call_ids":["senseval3.d002.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nervous%5:00:00:troubled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001561,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5694999169791117}
{"item_id":"senseval3.d002.s034.t001","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s034.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002676,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0564505840302445}
{"item_id":"senseval3.d002.s035.t001","gold_sense_keys":["fill%1:27:00::"],"candidates":[{"index":1,"sense_key":"fill%1:23:00::","synset_id":"fill.n.01"},{"index":2,"sense_key":"fill%1:27:00::","synset_id":"filling.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fill%1:27:00::","call_ids":["senseval3.d002.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fill%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":299,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040535,"input_uncached_usd":0.00001495,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.937469249998685}
{"item_id":"senseval3.d002.s035.t002","gold_sense_keys":["liquefy%2:30:00::","liquefy%2:43:00::"],"candidates":[{"index":1,"sense_key":"liquefy%2:43:00::","synset_id":"liquefy.v.01"},{"index":2,"sense_key":"liquefy%2:30:01::","synset_id":"liquefy.v.02"},{"index":3,"sense_key":"liquefy%2:30:00::","synset_id":"liquefy.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"liquefy%2:43:00::","call_ids":["senseval3.d002.s035.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"liquefy%2:43:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025475,"input_uncached_usd":0.00001795,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.75003045797348}
{"item_id":"senseval3.d002.s036.t000","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval3.d002.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018114999999999999,"input_uncached_usd":0.00002115,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6683728329953738}
{"item_id":"senseval3.d002.s036.t003","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"able%5:00:00:capable:00","call_ids":["senseval3.d002.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"able%5:00:00:capable:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003086,"input_uncached_usd":0.0000206,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3326908749877475}
{"item_id":"senseval3.d002.s036.t004","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk%2:38:00::","call_ids":["senseval3.d002.s036.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036749999999999993,"input_uncached_usd":0.0000283,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.7767695839866064}
{"item_id":"senseval3.d002.s036.t005","gold_sense_keys":["window%1:06:00::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"window%1:06:05::","call_ids":["senseval3.d002.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"window%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007253,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.00924808299169}
{"item_id":"senseval3.d002.s036.t007","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s036.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021715,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.680872333992738}
{"item_id":"senseval3.d002.s036.t008","gold_sense_keys":["street%1:06:00::"],"candidates":[{"index":1,"sense_key":"street%1:06:00::","synset_id":"street.n.01"},{"index":2,"sense_key":"street%1:06:01::","synset_id":"street.n.02"},{"index":3,"sense_key":"street%1:26:00::","synset_id":"street.n.03"},{"index":4,"sense_key":"street%1:26:01::","synset_id":"street.n.04"},{"index":5,"sense_key":"street%1:14:00::","synset_id":"street.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"street%1:06:01::","call_ids":["senseval3.d002.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"street%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002845,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.850239208957646}
{"item_id":"senseval3.d002.s036.t009","gold_sense_keys":["level%1:07:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"level%1:06:03::","call_ids":["senseval3.d002.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"level%1:06:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":2001,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00082975,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.0008003999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.79619962500874}
{"item_id":"senseval3.d002.s037.t000","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045085,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.854952916968614}
{"item_id":"senseval3.d002.s037.t001","gold_sense_keys":["ground%1:17:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":3,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":4,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":5,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":6,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":7,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":8,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":11,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"ground%1:17:01::","call_ids":["senseval3.d002.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"ground%1:17:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045315,"input_uncached_usd":0.000037149999999999995,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.927233207970858}
{"item_id":"senseval3.d002.s040.t000","gold_sense_keys":["drive%2:38:01::"],"candidates":[{"index":1,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":2,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":3,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":4,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":5,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":6,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":7,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":8,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":9,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":10,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":11,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":12,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":13,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":14,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":15,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":16,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":17,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":18,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":19,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":20,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":21,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":22,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drive%2:38:01::","call_ids":["senseval3.d002.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drive%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1116,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00039499999999999995,"input_uncached_usd":0.000055799999999999994,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.370632332982495}
{"item_id":"senseval3.d002.s040.t001","gold_sense_keys":["truck%1:06:00::"],"candidates":[{"index":1,"sense_key":"truck%1:06:00::","synset_id":"truck.n.01"},{"index":2,"sense_key":"truck%1:06:01::","synset_id":"hand_truck.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"truck%1:06:00::","call_ids":["senseval3.d002.s040.t001__v1__a1","senseval3.d002.s040.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"truck%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045814999999999994,"input_uncached_usd":0.00003575,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.172170665988233}
{"item_id":"senseval3.d002.s040.t002","gold_sense_keys":["stop%2:38:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stop%2:38:00::","call_ids":["senseval3.d002.s040.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stop%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017304999999999998,"input_uncached_usd":0.00003865,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.988187749986537}
{"item_id":"senseval3.d002.s040.t003","gold_sense_keys":["red_light%1:10:00::"],"candidates":[{"index":1,"sense_key":"red_light%1:10:01::","synset_id":"red_light.n.01"},{"index":2,"sense_key":"red_light%1:10:00::","synset_id":"red_light.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"red_light%1:10:00::","call_ids":["senseval3.d002.s040.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"red_light%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002777,"input_uncached_usd":0.0000153,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.285894916974939}
{"item_id":"senseval3.d002.s040.t004","gold_sense_keys":["corner%1:06:00::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"corner%1:06:00::","call_ids":["senseval3.d002.s040.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"corner%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019265,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.305019458988681}
{"item_id":"senseval3.d002.s040.t006","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"hit%2:35:05::","call_ids":["senseval3.d002.s040.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"hit%2:35:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":991,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00087515,"input_uncached_usd":0.00004955,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.851996417040937}
{"item_id":"senseval3.d002.s041.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["senseval3.d002.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007653,"input_uncached_usd":0.0000421,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.301411957945675}
{"item_id":"senseval3.d002.s041.t001","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039275,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.572796709020622}
{"item_id":"senseval3.d002.s041.t003","gold_sense_keys":["wave%2:38:00::"],"candidates":[{"index":1,"sense_key":"wave%2:32:00::","synset_id":"beckon.v.01"},{"index":2,"sense_key":"wave%2:35:00::","synset_id":"brandish.v.01"},{"index":3,"sense_key":"wave%2:38:00::","synset_id":"roll.v.11"},{"index":4,"sense_key":"wave%2:35:01::","synset_id":"curl.v.04"},{"index":5,"sense_key":"wave%2:29:00::","synset_id":"wave.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wave%2:38:00::","call_ids":["senseval3.d002.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wave%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001314,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9783562920056283}
{"item_id":"senseval3.d002.s041.t004","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1533,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00023664999999999998,"input_uncached_usd":0.00007664999999999999,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.120810457970947}
{"item_id":"senseval3.d002.s041.t005","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["senseval3.d002.s041.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004008,"input_uncached_usd":0.000036,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.551935625029728}
{"item_id":"senseval3.d002.s041.t006","gold_sense_keys":["drive%2:38:00::"],"candidates":[{"index":1,"sense_key":"drive%2:38:01::","synset_id":"drive.v.01"},{"index":2,"sense_key":"drive%2:38:00::","synset_id":"drive.v.02"},{"index":3,"sense_key":"drive%2:38:02::","synset_id":"drive.v.03"},{"index":4,"sense_key":"drive%2:35:00::","synset_id":"force.v.06"},{"index":5,"sense_key":"drive%2:41:02::","synset_id":"drive.v.05"},{"index":6,"sense_key":"drive%2:35:11::","synset_id":"repel.v.01"},{"index":7,"sense_key":"drive%2:36:00::","synset_id":"drive.v.07"},{"index":8,"sense_key":"drive%2:35:07::","synset_id":"drive.v.08"},{"index":9,"sense_key":"drive%2:35:01::","synset_id":"drive.v.09"},{"index":10,"sense_key":"drive%2:41:00::","synset_id":"tug.v.02"},{"index":11,"sense_key":"drive%2:32:00::","synset_id":"drive.v.11"},{"index":12,"sense_key":"drive%2:42:00::","synset_id":"drive.v.12"},{"index":13,"sense_key":"drive%2:41:13::","synset_id":"drive.v.13"},{"index":14,"sense_key":"drive%2:38:09::","synset_id":"drive.v.14"},{"index":15,"sense_key":"drive%2:38:10::","synset_id":"drive.v.15"},{"index":16,"sense_key":"drive%2:38:11::","synset_id":"drive.v.16"},{"index":17,"sense_key":"drive%2:35:03::","synset_id":"drive.v.17"},{"index":18,"sense_key":"drive%2:35:02::","synset_id":"drive.v.18"},{"index":19,"sense_key":"drive%2:35:06::","synset_id":"drive.v.19"},{"index":20,"sense_key":"drive%2:34:07::","synset_id":"drive.v.20"},{"index":21,"sense_key":"drive%2:33:01::","synset_id":"drive.v.21"},{"index":22,"sense_key":"drive%2:33:00::","synset_id":"drive.v.22"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drive%2:38:01::","call_ids":["senseval3.d002.s041.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drive%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004207,"input_uncached_usd":0.0000555,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9663120830082335}
{"item_id":"senseval3.d002.s044.t000","gold_sense_keys":["foot%1:23:00::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foot%1:23:00::","call_ids":["senseval3.d002.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019454999999999999,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.025423665996641}
{"item_id":"senseval3.d002.s044.t001","gold_sense_keys":["pier%1:06:00::"],"candidates":[{"index":1,"sense_key":"pier%1:06:00::","synset_id":"pier.n.01"},{"index":2,"sense_key":"pier%1:06:02::","synset_id":"pier.n.02"},{"index":3,"sense_key":"pier%1:06:01::","synset_id":"pier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pier%1:06:00::","call_ids":["senseval3.d002.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pier%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":339,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017735,"input_uncached_usd":0.00001695,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6825532920192927}
{"item_id":"senseval3.d002.s045.t001","gold_sense_keys":["dramatic%5:00:00:impressive:00"],"candidates":[{"index":1,"sense_key":"dramatic%3:00:00::","synset_id":"dramatic.a.01"},{"index":2,"sense_key":"dramatic%5:00:00:impressive:00","synset_id":"dramatic.s.01"},{"index":3,"sense_key":"dramatic%3:01:00::","synset_id":"dramatic.a.03"},{"index":4,"sense_key":"dramatic%3:00:02::","synset_id":"dramatic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dramatic%5:00:00:impressive:00","call_ids":["senseval3.d002.s045.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dramatic%5:00:00:impressive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036004999999999994,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.280647500010673}
{"item_id":"senseval3.d002.s046.t001","gold_sense_keys":["hairline%1:10:00::"],"candidates":[{"index":1,"sense_key":"hairline%1:10:00::","synset_id":"hairline.n.01"},{"index":2,"sense_key":"hairline%1:08:00::","synset_id":"hairline.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hairline%1:10:00::","call_ids":["senseval3.d002.s046.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hairline%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":265,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017324999999999998,"input_uncached_usd":0.00001325,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8723918750183657}
{"item_id":"senseval3.d002.s046.t002","gold_sense_keys":["crack%1:07:00::"],"candidates":[{"index":1,"sense_key":"crack%1:17:00::","synset_id":"crack.n.01"},{"index":2,"sense_key":"crack%1:06:00::","synset_id":"gap.n.03"},{"index":3,"sense_key":"crack%1:25:00::","synset_id":"crevice.n.01"},{"index":4,"sense_key":"crack%1:11:00::","synset_id":"crack.n.04"},{"index":5,"sense_key":"crack%1:26:00::","synset_id":"shot.n.04"},{"index":6,"sense_key":"crack%1:10:00::","synset_id":"wisecrack.n.01"},{"index":7,"sense_key":"crack%1:07:00::","synset_id":"crack.n.07"},{"index":8,"sense_key":"crack%1:06:01::","synset_id":"crack.n.08"},{"index":9,"sense_key":"crack%1:04:01::","synset_id":"crack.n.09"},{"index":10,"sense_key":"crack%1:04:00::","synset_id":"fracture.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"crack%1:07:00::","call_ids":["senseval3.d002.s046.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"crack%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057375,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.719561374979094}
{"item_id":"senseval3.d002.s047.t000","gold_sense_keys":["ruin%2:41:00::"],"candidates":[{"index":1,"sense_key":"ruin%2:35:00::","synset_id":"destroy.v.02"},{"index":2,"sense_key":"ruin%2:41:00::","synset_id":"ruin.v.02"},{"index":3,"sense_key":"ruin%2:40:00::","synset_id":"bankrupt.v.01"},{"index":4,"sense_key":"ruin%2:35:01::","synset_id":"ruin.v.04"},{"index":5,"sense_key":"ruin%2:35:02::","synset_id":"deflower.v.01"},{"index":6,"sense_key":"ruin%2:30:00::","synset_id":"ruin.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"ruin%2:41:00::","call_ids":["senseval3.d002.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"ruin%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028445,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.295709875004832}
{"item_id":"senseval3.d002.s047.t001","gold_sense_keys":["damn%5:00:02:cursed:00"],"candidates":[{"index":1,"sense_key":"damn%5:00:00:cursed:00","synset_id":"damn.s.01"},{"index":2,"sense_key":"damn%5:00:02:cursed:00","synset_id":"blasted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damn%5:00:02:cursed:00","call_ids":["senseval3.d002.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damn%5:00:02:cursed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":307,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053375,"input_uncached_usd":0.00001535,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.875424958008807}
{"item_id":"senseval3.d002.s050.t001","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["senseval3.d002.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":303,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030315,"input_uncached_usd":0.000015149999999999999,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.023199916991871}
{"item_id":"senseval3.d002.s050.t002","gold_sense_keys":["floor%1:06:01::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"floor%1:06:01::","call_ids":["senseval3.d002.s050.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"floor%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021624999999999997,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0185768329538405}
{"item_id":"senseval3.d002.s051.t000","gold_sense_keys":["hear%2:39:00::"],"candidates":[{"index":1,"sense_key":"hear%2:39:00::","synset_id":"hear.v.01"},{"index":2,"sense_key":"hear%2:31:00::","synset_id":"learn.v.02"},{"index":3,"sense_key":"hear%2:41:00::","synset_id":"hear.v.03"},{"index":4,"sense_key":"hear%2:39:01::","synset_id":"hear.v.04"},{"index":5,"sense_key":"hear%2:39:05::","synset_id":"listen.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hear%2:39:00::","call_ids":["senseval3.d002.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hear%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002844,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.285658499982674}
{"item_id":"senseval3.d002.s051.t001","gold_sense_keys":["part%1:06:00::","part%1:15:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["senseval3.d002.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027715,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.649897499999497}
{"item_id":"senseval3.d002.s051.t002","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s051.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001803,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0686852920334786}
{"item_id":"senseval3.d002.s051.t003","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"head%1:15:00::","call_ids":["senseval3.d002.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"head%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1498,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00023489999999999996,"input_uncached_usd":0.00007489999999999999,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.995893082988914}
{"item_id":"senseval3.d002.s051.t004","gold_sense_keys":["crack%2:30:00::","crack%2:30:01::","crack%2:39:00::","crack%2:39:01::"],"candidates":[{"index":1,"sense_key":"crack%2:30:01::","synset_id":"crack.v.01"},{"index":2,"sense_key":"crack%2:39:01::","synset_id":"crack.v.02"},{"index":3,"sense_key":"crack%2:39:00::","synset_id":"snap.v.06"},{"index":4,"sense_key":"crack%2:35:00::","synset_id":"crack.v.04"},{"index":5,"sense_key":"crack%2:30:09::","synset_id":"break_through.v.01"},{"index":6,"sense_key":"crack%2:30:02::","synset_id":"crack.v.06"},{"index":7,"sense_key":"crack%2:30:00::","synset_id":"snap.v.03"},{"index":8,"sense_key":"crack%2:41:00::","synset_id":"crack.v.08"},{"index":9,"sense_key":"crack%2:37:00::","synset_id":"crack_up.v.01"},{"index":10,"sense_key":"crack%2:32:00::","synset_id":"crack.v.10"},{"index":11,"sense_key":"crack%2:30:05::","synset_id":"crack.v.11"},{"index":12,"sense_key":"crack%2:30:07::","synset_id":"crack.v.12"},{"index":13,"sense_key":"crack%2:30:08::","synset_id":"crack.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"crack%2:30:02::","call_ids":["senseval3.d002.s051.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"crack%2:30:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042395,"input_uncached_usd":0.000033549999999999996,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.694858041999396}
{"item_id":"senseval3.d002.s052.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d002.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022514999999999997,"input_uncached_usd":0.00003955,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.568220083019696}
{"item_id":"senseval3.d002.s052.t001","gold_sense_keys":["die%2:30:00::"],"candidates":[{"index":1,"sense_key":"die%2:30:00::","synset_id":"die.v.01"},{"index":2,"sense_key":"die%2:39:00::","synset_id":"die.v.02"},{"index":3,"sense_key":"die%2:37:00::","synset_id":"die.v.03"},{"index":4,"sense_key":"die%2:30:04::","synset_id":"fail.v.04"},{"index":5,"sense_key":"die%2:37:03::","synset_id":"die.v.05"},{"index":6,"sense_key":"die%2:37:01::","synset_id":"die.v.06"},{"index":7,"sense_key":"die%2:35:00::","synset_id":"die.v.07"},{"index":8,"sense_key":"die%2:33:00::","synset_id":"die.v.08"},{"index":9,"sense_key":"die%2:30:05::","synset_id":"die.v.09"},{"index":10,"sense_key":"die%2:30:01::","synset_id":"die.v.10"},{"index":11,"sense_key":"die%2:30:02::","synset_id":"die.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"die%2:30:00::","call_ids":["senseval3.d002.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"die%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037374999999999995,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.510163375001866}
{"item_id":"senseval3.d002.s053.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d002.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038445,"input_uncached_usd":0.00001965,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2615939999814145}
{"item_id":"senseval3.d002.s053.t001","gold_sense_keys":["come%2:38:00::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d002.s053.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036839999999999996,"input_uncached_usd":0.0000548,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.113716749998275}
{"item_id":"senseval3.d002.s053.t002","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003454,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.937420457950793}
{"item_id":"senseval3.d002.s053.t003","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s053.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037404999999999996,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.014867957972456}
{"item_id":"senseval3.d002.s053.t004","gold_sense_keys":["stilt%1:06:01::"],"candidates":[{"index":1,"sense_key":"stilt%1:06:01::","synset_id":"pile.n.06"},{"index":2,"sense_key":"stilt%1:06:00::","synset_id":"stilt.n.02"},{"index":3,"sense_key":"stilt%1:05:02::","synset_id":"stilt.n.03"},{"index":4,"sense_key":"stilt%1:05:01::","synset_id":"stilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stilt%1:06:01::","call_ids":["senseval3.d002.s053.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stilt%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033469999999999995,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.433673499966972}
{"item_id":"senseval3.d002.s054.t000","gold_sense_keys":["decide%2:31:00::"],"candidates":[{"index":1,"sense_key":"decide%2:31:00::","synset_id":"decide.v.01"},{"index":2,"sense_key":"decide%2:31:01::","synset_id":"decide.v.02"},{"index":3,"sense_key":"decide%2:31:02::","synset_id":"decide.v.03"},{"index":4,"sense_key":"decide%2:31:06::","synset_id":"decide.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decide%2:31:00::","call_ids":["senseval3.d002.s054.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decide%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020605,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.1380890409927815}
{"item_id":"senseval3.d002.s055.t001","gold_sense_keys":["smell%1:07:00::","smell%1:09:02::"],"candidates":[{"index":1,"sense_key":"smell%1:09:02::","synset_id":"smell.n.01"},{"index":2,"sense_key":"smell%1:07:00::","synset_id":"olfactory_property.n.01"},{"index":3,"sense_key":"smell%1:26:00::","synset_id":"spirit.n.02"},{"index":4,"sense_key":"smell%1:09:01::","synset_id":"smell.n.04"},{"index":5,"sense_key":"smell%1:04:00::","synset_id":"smell.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"smell%1:07:00::","call_ids":["senseval3.d002.s055.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"smell%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004411,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.455247790960129}
{"item_id":"senseval3.d002.s055.t002","gold_sense_keys":["gas%1:27:02::"],"candidates":[{"index":1,"sense_key":"gas%1:26:00::","synset_id":"gas.n.01"},{"index":2,"sense_key":"gas%1:27:00::","synset_id":"gas.n.02"},{"index":3,"sense_key":"gas%1:27:02::","synset_id":"gasoline.n.01"},{"index":4,"sense_key":"gas%1:26:01::","synset_id":"flatulence.n.01"},{"index":5,"sense_key":"gas%1:06:00::","synset_id":"accelerator.n.01"},{"index":6,"sense_key":"gas%1:27:03::","synset_id":"natural_gas.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"gas%1:27:02::","call_ids":["senseval3.d002.s055.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"gas%1:27:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004431,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.638178374967538}
{"item_id":"senseval3.d002.s055.t003","gold_sense_keys":["pass%2:38:05::"],"candidates":[{"index":1,"sense_key":"pass%2:38:00::","synset_id":"pass.v.01"},{"index":2,"sense_key":"pass%2:38:05::","synset_id":"travel_by.v.01"},{"index":3,"sense_key":"pass%2:41:02::","synset_id":"legislate.v.01"},{"index":4,"sense_key":"pass%2:38:03::","synset_id":"elapse.v.01"},{"index":5,"sense_key":"pass%2:40:00::","synset_id":"pass.v.05"},{"index":6,"sense_key":"pass%2:42:08::","synset_id":"run.v.03"},{"index":7,"sense_key":"pass%2:38:01::","synset_id":"pass.v.07"},{"index":8,"sense_key":"pass%2:30:00::","synset_id":"happen.v.01"},{"index":9,"sense_key":"pass%2:41:04::","synset_id":"pass.v.09"},{"index":10,"sense_key":"pass%2:42:00::","synset_id":"spend.v.01"},{"index":11,"sense_key":"pass%2:35:00::","synset_id":"guide.v.05"},{"index":12,"sense_key":"pass%2:32:01::","synset_id":"communicate.v.01"},{"index":13,"sense_key":"pass%2:30:01::","synset_id":"evanesce.v.01"},{"index":14,"sense_key":"pass%2:41:08::","synset_id":"pass.v.14"},{"index":15,"sense_key":"pass%2:42:04::","synset_id":"exceed.v.02"},{"index":16,"sense_key":"pass%2:41:07::","synset_id":"pass.v.16"},{"index":17,"sense_key":"pass%2:41:11::","synset_id":"pass.v.17"},{"index":18,"sense_key":"pass%2:40:02::","synset_id":"pass.v.18"},{"index":19,"sense_key":"pass%2:38:02::","synset_id":"sink.v.03"},{"index":20,"sense_key":"pass%2:40:01::","synset_id":"pass.v.20"},{"index":21,"sense_key":"pass%2:40:13::","synset_id":"fall.v.21"},{"index":22,"sense_key":"pass%2:38:04::","synset_id":"pass.v.22"},{"index":23,"sense_key":"pass%2:32:00::","synset_id":"authorize.v.01"},{"index":24,"sense_key":"pass%2:30:11::","synset_id":"die.v.01"},{"index":25,"sense_key":"pass%2:29:00::","synset_id":"excrete.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pass%2:38:05::","call_ids":["senseval3.d002.s055.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pass%2:38:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1395,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00038334999999999997,"input_uncached_usd":0.00006975,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.853456083976198}
{"item_id":"senseval3.d002.s055.t005","gold_sense_keys":["cross%2:38:00::"],"candidates":[{"index":1,"sense_key":"cross%2:38:00::","synset_id":"traverse.v.01"},{"index":2,"sense_key":"cross%2:38:03::","synset_id":"intersect.v.01"},{"index":3,"sense_key":"cross%2:41:00::","synset_id":"thwart.v.01"},{"index":4,"sense_key":"cross%2:38:04::","synset_id":"cross.v.04"},{"index":5,"sense_key":"cross%2:42:00::","synset_id":"cross.v.05"},{"index":6,"sense_key":"cross%2:38:01::","synset_id":"cross.v.06"},{"index":7,"sense_key":"cross%2:36:09::","synset_id":"cross.v.07"},{"index":8,"sense_key":"cross%2:35:00::","synset_id":"crossbreed.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cross%2:38:00::","call_ids":["senseval3.d002.s055.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cross%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024119999999999998,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.134899874974508}
{"item_id":"senseval3.d002.s056.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"see%2:39:03::","call_ids":["senseval3.d002.s056.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"see%2:39:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1388,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00025499999999999996,"input_uncached_usd":0.00006939999999999999,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6153907499974594}
{"item_id":"senseval3.d002.s056.t001","gold_sense_keys":["cloud%1:19:01::"],"candidates":[{"index":1,"sense_key":"cloud%1:19:01::","synset_id":"cloud.n.01"},{"index":2,"sense_key":"cloud%1:17:00::","synset_id":"cloud.n.02"},{"index":3,"sense_key":"cloud%1:26:01::","synset_id":"cloud.n.03"},{"index":4,"sense_key":"cloud%1:26:02::","synset_id":"cloud.n.04"},{"index":5,"sense_key":"cloud%1:26:00::","synset_id":"cloud.n.05"},{"index":6,"sense_key":"cloud%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cloud%1:17:00::","call_ids":["senseval3.d002.s056.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cloud%1:17:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013179999999999998,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6944053749903105}
{"item_id":"senseval3.d002.s056.t002","gold_sense_keys":["bay%1:17:00::"],"candidates":[{"index":1,"sense_key":"bay%1:17:00::","synset_id":"bay.n.01"},{"index":2,"sense_key":"bay%1:11:00::","synset_id":"bay.n.02"},{"index":3,"sense_key":"bay%1:20:00::","synset_id":"true_laurel.n.01"},{"index":4,"sense_key":"bay%1:06:02::","synset_id":"bay.n.04"},{"index":5,"sense_key":"bay%1:06:01::","synset_id":"bay.n.05"},{"index":6,"sense_key":"bay%1:06:00::","synset_id":"alcove.n.01"},{"index":7,"sense_key":"bay%1:05:00::","synset_id":"bay.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bay%1:17:00::","call_ids":["senseval3.d002.s056.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bay%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010809999999999998,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.382548499968834}
{"item_id":"senseval3.d002.s056.t004","gold_sense_keys":["fire%1:11:00::"],"candidates":[{"index":1,"sense_key":"fire%1:11:00::","synset_id":"fire.n.01"},{"index":2,"sense_key":"fire%1:04:00::","synset_id":"fire.n.02"},{"index":3,"sense_key":"fire%1:22:00::","synset_id":"fire.n.03"},{"index":4,"sense_key":"fire%1:06:00::","synset_id":"fire.n.04"},{"index":5,"sense_key":"fire%1:27:01::","synset_id":"fire.n.05"},{"index":6,"sense_key":"fire%1:12:00::","synset_id":"ardor.n.03"},{"index":7,"sense_key":"fire%1:27:00::","synset_id":"fire.n.07"},{"index":8,"sense_key":"fire%1:11:01::","synset_id":"fire.n.08"},{"index":9,"sense_key":"fire%1:10:00::","synset_id":"fire.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fire%1:11:00::","call_ids":["senseval3.d002.s056.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fire%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024304999999999997,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6260147499851882}
{"item_id":"senseval3.d002.s057.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s057.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030195,"input_uncached_usd":0.00003955,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.169899375003297}
{"item_id":"senseval3.d002.s058.t000","gold_sense_keys":["back%1:08:00::"],"candidates":[{"index":1,"sense_key":"back%1:08:00::","synset_id":"back.n.01"},{"index":2,"sense_key":"back%1:06:00::","synset_id":"rear.n.05"},{"index":3,"sense_key":"back%1:15:02::","synset_id":"back.n.03"},{"index":4,"sense_key":"back%1:18:00::","synset_id":"back.n.04"},{"index":5,"sense_key":"back%1:08:01::","synset_id":"spinal_column.n.01"},{"index":6,"sense_key":"back%1:06:03::","synset_id":"binding.n.05"},{"index":7,"sense_key":"back%1:06:02::","synset_id":"back.n.07"},{"index":8,"sense_key":"back%1:06:01::","synset_id":"back.n.08"},{"index":9,"sense_key":"back%1:04:00::","synset_id":"back.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"back%1:08:00::","call_ids":["senseval3.d002.s058.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"back%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032105,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.10450174997095}
{"item_id":"senseval3.d002.s058.t002","gold_sense_keys":["hand%1:08:00::"],"candidates":[{"index":1,"sense_key":"hand%1:08:00::","synset_id":"hand.n.01"},{"index":2,"sense_key":"hand%1:18:00::","synset_id":"hired_hand.n.01"},{"index":3,"sense_key":"hand%1:10:00::","synset_id":"handwriting.n.01"},{"index":4,"sense_key":"hand%1:09:00::","synset_id":"hand.n.04"},{"index":5,"sense_key":"hand%1:15:00::","synset_id":"hand.n.05"},{"index":6,"sense_key":"hand%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"hand%1:09:01::","synset_id":"hand.n.07"},{"index":8,"sense_key":"hand%1:06:00::","synset_id":"hand.n.08"},{"index":9,"sense_key":"hand%1:23:00::","synset_id":"hand.n.09"},{"index":10,"sense_key":"hand%1:18:02::","synset_id":"hand.n.10"},{"index":11,"sense_key":"hand%1:18:01::","synset_id":"bridge_player.n.01"},{"index":12,"sense_key":"hand%1:10:01::","synset_id":"hand.n.12"},{"index":13,"sense_key":"hand%1:05:01::","synset_id":"hand.n.13"},{"index":14,"sense_key":"hand%1:04:00::","synset_id":"hand.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hand%1:08:00::","call_ids":["senseval3.d002.s058.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hand%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":842,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022769999999999998,"input_uncached_usd":0.0000421,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.160292082990054}
{"item_id":"senseval3.d002.s058.t003","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shake%2:29:00::","call_ids":["senseval3.d002.s058.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shake%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":646,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003715,"input_uncached_usd":0.0000323,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.682502417010255}
{"item_id":"senseval3.d002.s059.t000","gold_sense_keys":["think%2:31:03::"],"candidates":[{"index":1,"sense_key":"think%2:31:01::","synset_id":"think.v.01"},{"index":2,"sense_key":"think%2:31:03::","synset_id":"think.v.02"},{"index":3,"sense_key":"think%2:31:00::","synset_id":"think.v.03"},{"index":4,"sense_key":"think%2:31:02::","synset_id":"remember.v.01"},{"index":5,"sense_key":"think%2:36:00::","synset_id":"think.v.05"},{"index":6,"sense_key":"think%2:31:09::","synset_id":"think.v.06"},{"index":7,"sense_key":"think%2:31:06::","synset_id":"intend.v.01"},{"index":8,"sense_key":"think%2:31:12::","synset_id":"think.v.08"},{"index":9,"sense_key":"think%2:31:13::","synset_id":"think.v.09"},{"index":10,"sense_key":"think%2:31:15::","synset_id":"think.v.10"},{"index":11,"sense_key":"think%2:31:14::","synset_id":"think.v.11"},{"index":12,"sense_key":"think%2:31:08::","synset_id":"think.v.12"},{"index":13,"sense_key":"think%2:30:00::","synset_id":"think.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"think%2:31:03::","call_ids":["senseval3.d002.s059.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"think%2:31:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003541,"input_uncached_usd":0.000040499999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.632091042003594}
{"item_id":"senseval3.d002.s059.t003","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["senseval3.d002.s059.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039935,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.179458209022414}
{"item_id":"senseval3.d002.s059.t004","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"shake%2:29:00::","call_ids":["senseval3.d002.s059.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"shake%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026849999999999997,"input_uncached_usd":0.0000317,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0239217090420425}
{"item_id":"senseval3.d002.s062.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s062.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1365,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00020264999999999997,"input_uncached_usd":0.00006824999999999999,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9036544589907862}
{"item_id":"senseval3.d002.s062.t002","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"house%1:14:04::","call_ids":["senseval3.d002.s062.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"house%1:14:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055315,"input_uncached_usd":0.00003475,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.499213666014839}
{"item_id":"senseval3.d002.s062.t003","gold_sense_keys":["bay%1:17:00::"],"candidates":[{"index":1,"sense_key":"bay%1:17:00::","synset_id":"bay.n.01"},{"index":2,"sense_key":"bay%1:11:00::","synset_id":"bay.n.02"},{"index":3,"sense_key":"bay%1:20:00::","synset_id":"true_laurel.n.01"},{"index":4,"sense_key":"bay%1:06:02::","synset_id":"bay.n.04"},{"index":5,"sense_key":"bay%1:06:01::","synset_id":"bay.n.05"},{"index":6,"sense_key":"bay%1:06:00::","synset_id":"alcove.n.01"},{"index":7,"sense_key":"bay%1:05:00::","synset_id":"bay.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bay%1:17:00::","call_ids":["senseval3.d002.s062.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bay%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002093,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.114981916965917}
{"item_id":"senseval3.d002.s063.t000","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s063.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024859999999999997,"input_uncached_usd":0.0000374,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.794136417040136}
{"item_id":"senseval3.d002.s063.t001","gold_sense_keys":["believe%2:31:00::"],"candidates":[{"index":1,"sense_key":"believe%2:31:00::","synset_id":"believe.v.01"},{"index":2,"sense_key":"believe%2:31:04::","synset_id":"think.v.01"},{"index":3,"sense_key":"believe%2:31:03::","synset_id":"believe.v.03"},{"index":4,"sense_key":"believe%2:31:01::","synset_id":"believe.v.04"},{"index":5,"sense_key":"believe%2:31:02::","synset_id":"believe.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"believe%2:31:00::","call_ids":["senseval3.d002.s063.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"believe%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020759999999999998,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.008988791960292}
{"item_id":"senseval3.d002.s063.t002","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":3,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":4,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":5,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d002.s063.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033769999999999997,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.253283375001047}
{"item_id":"senseval3.d002.s066.t000","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s066.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023104999999999998,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.108623041014653}
{"item_id":"senseval3.d002.s066.t001","gold_sense_keys":["corner%1:06:00::"],"candidates":[{"index":1,"sense_key":"corner%1:15:02::","synset_id":"corner.n.01"},{"index":2,"sense_key":"corner%1:15:00::","synset_id":"corner.n.02"},{"index":3,"sense_key":"corner%1:06:02::","synset_id":"corner.n.03"},{"index":4,"sense_key":"corner%1:06:00::","synset_id":"corner.n.04"},{"index":5,"sense_key":"corner%1:15:01::","synset_id":"corner.n.05"},{"index":6,"sense_key":"corner%1:25:00::","synset_id":"recess.n.02"},{"index":7,"sense_key":"corner%1:26:00::","synset_id":"corner.n.07"},{"index":8,"sense_key":"corner%1:26:01::","synset_id":"corner.n.08"},{"index":9,"sense_key":"corner%1:17:00::","synset_id":"corner.n.09"},{"index":10,"sense_key":"corner%1:15:03::","synset_id":"corner.n.10"},{"index":11,"sense_key":"corner%1:06:01::","synset_id":"corner.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"corner%1:06:00::","call_ids":["senseval3.d002.s066.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"corner%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006002000000000001,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.925444875028916}
{"item_id":"senseval3.d002.s066.t002","gold_sense_keys":["damage%2:30:00::"],"candidates":[{"index":1,"sense_key":"damage%2:30:00::","synset_id":"damage.v.01"},{"index":2,"sense_key":"damage%2:30:01::","synset_id":"damage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"damage%2:30:01::","call_ids":["senseval3.d002.s066.t002__v1__a1","senseval3.d002.s066.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"damage%2:30:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045315,"input_uncached_usd":0.00003075,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.597278624074534}
{"item_id":"senseval3.d002.s066.t003","gold_sense_keys":["old%3:00:02::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"old%3:00:02::","call_ids":["senseval3.d002.s066.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"old%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016375,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.015430458006449}
{"item_id":"senseval3.d002.s066.t004","gold_sense_keys":["lady%1:18:02::"],"candidates":[{"index":1,"sense_key":"lady%1:18:02::","synset_id":"lady.n.01"},{"index":2,"sense_key":"lady%1:18:01::","synset_id":"dame.n.02"},{"index":3,"sense_key":"lady%1:18:00::","synset_id":"lady.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lady%1:18:02::","call_ids":["senseval3.d002.s066.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lady%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":324,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001762,"input_uncached_usd":0.0000162,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3834155420190655}
{"item_id":"senseval3.d002.s066.t005","gold_sense_keys":["old%3:00:02::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"old%3:00:02::","call_ids":["senseval3.d002.s066.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"old%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021535,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.655097750015557}
{"item_id":"senseval3.d002.s066.t006","gold_sense_keys":["mother%1:18:00::"],"candidates":[{"index":1,"sense_key":"mother%1:18:00::","synset_id":"mother.n.01"},{"index":2,"sense_key":"mother%1:27:00::","synset_id":"mother.n.02"},{"index":3,"sense_key":"mother%1:18:02::","synset_id":"mother.n.03"},{"index":4,"sense_key":"mother%1:18:01::","synset_id":"mother.n.04"},{"index":5,"sense_key":"mother%1:09:00::","synset_id":"mother.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mother%1:18:00::","call_ids":["senseval3.d002.s066.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mother%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005635,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.307649000023957}
{"item_id":"senseval3.d002.s067.t000","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s067.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024609999999999996,"input_uncached_usd":0.0000349,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.028515083016828}
{"item_id":"senseval3.d002.s068.t000","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["senseval3.d002.s068.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":285,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030225,"input_uncached_usd":0.000014249999999999999,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.064650957996491}
{"item_id":"senseval3.d002.s068.t001","gold_sense_keys":["standing%3:00:00::"],"candidates":[{"index":1,"sense_key":"standing%5:00:00:erect:00","synset_id":"standing.s.01"},{"index":2,"sense_key":"standing%5:00:00:permanent:00","synset_id":"standing.s.02"},{"index":3,"sense_key":"standing%3:00:02::","synset_id":"standing.a.03"},{"index":4,"sense_key":"standing%3:00:01::","synset_id":"standing.a.04"},{"index":5,"sense_key":"standing%3:00:00::","synset_id":"standing.a.05"},{"index":6,"sense_key":"standing%5:00:00:regular:02","synset_id":"standing.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"standing%3:00:00::","call_ids":["senseval3.d002.s068.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"standing%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001805,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.828886458009947}
{"item_id":"senseval3.d002.s068.t002","gold_sense_keys":["position%1:07:01::"],"candidates":[{"index":1,"sense_key":"position%1:15:00::","synset_id":"position.n.01"},{"index":2,"sense_key":"position%1:15:02::","synset_id":"military_position.n.01"},{"index":3,"sense_key":"position%1:09:00::","synset_id":"position.n.03"},{"index":4,"sense_key":"position%1:07:01::","synset_id":"position.n.04"},{"index":5,"sense_key":"position%1:26:00::","synset_id":"status.n.01"},{"index":6,"sense_key":"position%1:04:01::","synset_id":"position.n.06"},{"index":7,"sense_key":"position%1:07:00::","synset_id":"position.n.07"},{"index":8,"sense_key":"position%1:15:03::","synset_id":"position.n.08"},{"index":9,"sense_key":"position%1:04:02::","synset_id":"position.n.09"},{"index":10,"sense_key":"position%1:04:00::","synset_id":"placement.n.03"},{"index":11,"sense_key":"position%1:26:02::","synset_id":"situation.n.02"},{"index":12,"sense_key":"position%1:09:01::","synset_id":"position.n.12"},{"index":13,"sense_key":"position%1:10:00::","synset_id":"side.n.10"},{"index":14,"sense_key":"position%1:10:01::","synset_id":"place.n.15"},{"index":15,"sense_key":"position%1:04:03::","synset_id":"stead.n.01"},{"index":16,"sense_key":"position%1:04:04::","synset_id":"position.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"position%1:07:01::","call_ids":["senseval3.d002.s068.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"position%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017824999999999997,"input_uncached_usd":0.000043849999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.97714591701515}
{"item_id":"senseval3.d002.s072.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s072.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045315,"input_uncached_usd":0.000037149999999999995,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.373785500007216}
{"item_id":"senseval3.d002.s072.t001","gold_sense_keys":["thing%1:06:01::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"thing%1:06:01::","call_ids":["senseval3.d002.s072.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"thing%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008297000000000001,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.572972666996066}
{"item_id":"senseval3.d002.s072.t002","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s072.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038475,"input_uncached_usd":0.00001995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.916896250040736}
{"item_id":"senseval3.d002.s072.t003","gold_sense_keys":["strange%3:00:00::"],"candidates":[{"index":1,"sense_key":"strange%3:00:00::","synset_id":"strange.a.01"},{"index":2,"sense_key":"strange%5:00:01:unfamiliar:00","synset_id":"strange.s.01"},{"index":3,"sense_key":"strange%3:00:01::","synset_id":"foreign.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strange%3:00:00::","call_ids":["senseval3.d002.s072.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strange%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000254,"input_uncached_usd":0.000017199999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.981252041005064}
{"item_id":"senseval3.d002.s072.t004","gold_sense_keys":["bird%1:05:00::"],"candidates":[{"index":1,"sense_key":"bird%1:05:00::","synset_id":"bird.n.01"},{"index":2,"sense_key":"bird%1:13:00::","synset_id":"bird.n.02"},{"index":3,"sense_key":"bird%1:18:00::","synset_id":"dame.n.01"},{"index":4,"sense_key":"bird%1:10:00::","synset_id":"boo.n.01"},{"index":5,"sense_key":"bird%1:06:01::","synset_id":"shuttlecock.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bird%1:05:00::","call_ids":["senseval3.d002.s072.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bird%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":978,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004111,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.00039119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.962618125020526}
{"item_id":"senseval3.d002.s072.t005","gold_sense_keys":["behavior%1:04:01::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"behavior%1:04:01::","call_ids":["senseval3.d002.s072.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"behavior%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020389999999999997,"input_uncached_usd":0.000018299999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.465131375007331}
{"item_id":"senseval3.d002.s073.t000","gold_sense_keys":["duck%1:05:00::"],"candidates":[{"index":1,"sense_key":"duck%1:05:00::","synset_id":"duck.n.01"},{"index":2,"sense_key":"duck%1:23:00::","synset_id":"duck.n.02"},{"index":3,"sense_key":"duck%1:13:00::","synset_id":"duck.n.03"},{"index":4,"sense_key":"duck%1:06:00::","synset_id":"duck.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"duck%1:05:00::","call_ids":["senseval3.d002.s073.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"duck%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025315,"input_uncached_usd":0.000016349999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7374589999672025}
{"item_id":"senseval3.d002.s073.t001","gold_sense_keys":["swarm%1:14:00::"],"candidates":[{"index":1,"sense_key":"swarm%1:14:01::","synset_id":"drove.n.02"},{"index":2,"sense_key":"swarm%1:14:00::","synset_id":"swarm.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"swarm%1:14:00::","call_ids":["senseval3.d002.s073.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"swarm%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":274,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006348999999999999,"input_uncached_usd":0.0000137,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.063174832961522}
{"item_id":"senseval3.d002.s076.t000","gold_sense_keys":["feel%2:39:00::"],"candidates":[{"index":1,"sense_key":"feel%2:37:00::","synset_id":"feel.v.01"},{"index":2,"sense_key":"feel%2:31:00::","synset_id":"find.v.05"},{"index":3,"sense_key":"feel%2:39:00::","synset_id":"feel.v.03"},{"index":4,"sense_key":"feel%2:29:00::","synset_id":"feel.v.04"},{"index":5,"sense_key":"feel%2:31:01::","synset_id":"feel.v.05"},{"index":6,"sense_key":"feel%2:39:01::","synset_id":"feel.v.06"},{"index":7,"sense_key":"feel%2:42:00::","synset_id":"feel.v.07"},{"index":8,"sense_key":"feel%2:39:10::","synset_id":"feel.v.08"},{"index":9,"sense_key":"feel%2:35:00::","synset_id":"feel.v.09"},{"index":10,"sense_key":"feel%2:35:01::","synset_id":"palpate.v.01"},{"index":11,"sense_key":"feel%2:40:00::","synset_id":"feel.v.11"},{"index":12,"sense_key":"feel%2:39:09::","synset_id":"feel.v.12"},{"index":13,"sense_key":"feel%2:35:11::","synset_id":"feel.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"feel%2:39:00::","call_ids":["senseval3.d002.s076.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"feel%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00012044999999999999,"input_uncached_usd":0.00003725,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.422835833975114}
{"item_id":"senseval3.d002.s076.t003","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s076.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036994999999999997,"input_uncached_usd":0.000030749999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.446720957988873}
{"item_id":"senseval3.d002.s077.t000","gold_sense_keys":["numb%5:00:00:afraid:00"],"candidates":[{"index":1,"sense_key":"numb%5:00:00:insensible:00","synset_id":"asleep.s.01"},{"index":2,"sense_key":"numb%5:00:00:insensitive:02","synset_id":"dead.s.04"},{"index":3,"sense_key":"numb%5:00:00:afraid:00","synset_id":"numb.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"numb%5:00:00:insensible:00","call_ids":["senseval3.d002.s077.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"numb%5:00:00:insensible:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061255,"input_uncached_usd":0.000017349999999999998,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.629245749965776}
{"item_id":"senseval3.d002.s080.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"seem%2:39:00::","call_ids":["senseval3.d002.s080.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"seem%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003084,"input_uncached_usd":0.000020399999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8719174999860115}
{"item_id":"senseval3.d002.s080.t002","gold_sense_keys":["part%1:09:00::","part%1:15:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"part%1:15:00::","call_ids":["senseval3.d002.s080.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"part%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":817,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020084999999999998,"input_uncached_usd":0.00004085,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.1283014170476235}
{"item_id":"senseval3.d002.s080.t003","gold_sense_keys":["town%1:15:00::"],"candidates":[{"index":1,"sense_key":"town%1:15:00::","synset_id":"town.n.01"},{"index":2,"sense_key":"town%1:14:00::","synset_id":"town.n.02"},{"index":3,"sense_key":"town%1:15:01::","synset_id":"township.n.01"},{"index":4,"sense_key":"town%1:18:00::","synset_id":"town.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"town%1:15:00::","call_ids":["senseval3.d002.s080.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"town%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023099999999999998,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.658811500004958}
{"item_id":"senseval3.d002.s081.t001","gold_sense_keys":["phone%1:06:00::"],"candidates":[{"index":1,"sense_key":"phone%1:06:00::","synset_id":"telephone.n.01"},{"index":2,"sense_key":"phone%1:10:00::","synset_id":"phone.n.02"},{"index":3,"sense_key":"phone%1:06:01::","synset_id":"earphone.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phone%1:06:00::","call_ids":["senseval3.d002.s081.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phone%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041135,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.272979250003118}
{"item_id":"senseval3.d002.s081.t002","gold_sense_keys":["mess%1:23:00::","mess%1:26:00::"],"candidates":[{"index":1,"sense_key":"mess%1:26:00::","synset_id":"mess.n.01"},{"index":2,"sense_key":"mess%1:26:02::","synset_id":"fix.n.01"},{"index":3,"sense_key":"mess%1:13:01::","synset_id":"mess.n.03"},{"index":4,"sense_key":"mess%1:13:00::","synset_id":"mess.n.04"},{"index":5,"sense_key":"mess%1:06:00::","synset_id":"mess.n.05"},{"index":6,"sense_key":"mess%1:23:00::","synset_id":"batch.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"mess%1:23:00::","call_ids":["senseval3.d002.s081.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"mess%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003901,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.54471695801476}
{"item_id":"senseval3.d002.s081.t004","gold_sense_keys":["wine%1:13:00::"],"candidates":[{"index":1,"sense_key":"wine%1:13:00::","synset_id":"wine.n.01"},{"index":2,"sense_key":"wine%1:07:00::","synset_id":"wine.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wine%1:13:00::","call_ids":["senseval3.d002.s081.t004__v1__a1","senseval3.d002.s081.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wine%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":667,"cached_input_tokens":0,"output_tokens":1825,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00076335,"input_uncached_usd":0.00003335,"input_cached_usd":0.0,"output_usd":0.00073,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.687223875022028}
{"item_id":"senseval3.d002.s081.t005","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s081.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016685,"input_uncached_usd":0.000032449999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6015108340070583}
{"item_id":"senseval3.d002.s081.t006","gold_sense_keys":["big%3:00:01::"],"candidates":[{"index":1,"sense_key":"big%3:00:01::","synset_id":"large.a.01"},{"index":2,"sense_key":"big%5:00:00:important:00","synset_id":"big.s.01"},{"index":3,"sense_key":"big%5:00:00:intense:00","synset_id":"bad.s.01"},{"index":4,"sense_key":"big%5:00:00:loud:00","synset_id":"big.s.03"},{"index":5,"sense_key":"big%5:00:00:conspicuous:00","synset_id":"big.s.04"},{"index":6,"sense_key":"big%5:00:00:intemperate:00","synset_id":"big.s.05"},{"index":7,"sense_key":"big%5:00:02:proud:00","synset_id":"boastful.s.01"},{"index":8,"sense_key":"big%5:00:00:proud:00","synset_id":"big.s.07"},{"index":9,"sense_key":"big%5:00:00:mature:01","synset_id":"adult.s.01"},{"index":10,"sense_key":"big%5:00:00:heavy:08","synset_id":"big.s.09"},{"index":11,"sense_key":"big%5:00:00:generous:02","synset_id":"big.s.10"},{"index":12,"sense_key":"big%5:00:00:generous:01","synset_id":"big.s.11"},{"index":13,"sense_key":"big%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"big%3:00:01::","call_ids":["senseval3.d002.s081.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"big%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027885,"input_uncached_usd":0.00004205,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.458216833998449}
{"item_id":"senseval3.d002.s081.t007","gold_sense_keys":["old%3:00:01::"],"candidates":[{"index":1,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":2,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":3,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":4,"sense_key":"old%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":5,"sense_key":"old%5:00:03:past:00","synset_id":"erstwhile.s.01"},{"index":6,"sense_key":"old%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":7,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":8,"sense_key":"old%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"old%3:00:01::","call_ids":["senseval3.d002.s081.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"old%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000268,"input_uncached_usd":0.0000312,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8157432500156574}
{"item_id":"senseval3.d002.s082.t000","gold_sense_keys":["quiver%1:26:00::"],"candidates":[{"index":1,"sense_key":"quiver%1:12:00::","synset_id":"frisson.n.01"},{"index":2,"sense_key":"quiver%1:26:00::","synset_id":"shaking.n.02"},{"index":3,"sense_key":"quiver%1:06:00::","synset_id":"quiver.n.03"},{"index":4,"sense_key":"quiver%1:04:00::","synset_id":"vibration.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"quiver%1:26:00::","call_ids":["senseval3.d002.s082.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"quiver%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035969999999999996,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.172404290991835}
{"item_id":"senseval3.d002.s082.t002","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s082.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003244,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.655803250032477}
{"item_id":"senseval3.d002.s082.t004","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s082.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001593,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0511254589655437}
{"item_id":"senseval3.d002.s082.t005","gold_sense_keys":["unpredictable%5:00:00:sporadic:00"],"candidates":[{"index":1,"sense_key":"unpredictable%3:00:00::","synset_id":"unpredictable.a.01"},{"index":2,"sense_key":"unpredictable%5:00:00:indeterminable:00","synset_id":"unpredictable.s.01"},{"index":3,"sense_key":"unpredictable%5:00:00:sporadic:00","synset_id":"irregular.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"unpredictable%5:00:00:sporadic:00","call_ids":["senseval3.d002.s082.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"unpredictable%5:00:00:sporadic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012644999999999998,"input_uncached_usd":0.00001765,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.160048291028943}
{"item_id":"senseval3.d002.s082.t006","gold_sense_keys":["interval%1:28:00::"],"candidates":[{"index":1,"sense_key":"interval%1:28:00::","synset_id":"time_interval.n.01"},{"index":2,"sense_key":"interval%1:09:00::","synset_id":"interval.n.02"},{"index":3,"sense_key":"interval%1:07:00::","synset_id":"interval.n.03"},{"index":4,"sense_key":"interval%1:10:00::","synset_id":"interval.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interval%1:28:00::","call_ids":["senseval3.d002.s082.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interval%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000767,"input_uncached_usd":0.0000191,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.4411045829765499}
{"item_id":"senseval3.d002.s082.t007","gold_sense_keys":["mouse%1:05:00::"],"candidates":[{"index":1,"sense_key":"mouse%1:05:00::","synset_id":"mouse.n.01"},{"index":2,"sense_key":"mouse%1:26:00::","synset_id":"shiner.n.01"},{"index":3,"sense_key":"mouse%1:18:00::","synset_id":"mouse.n.03"},{"index":4,"sense_key":"mouse%1:06:00::","synset_id":"mouse.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mouse%1:05:00::","call_ids":["senseval3.d002.s082.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mouse%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010469999999999998,"input_uncached_usd":0.000021499999999999997,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4971804159577005}
{"item_id":"senseval3.d002.s082.t008","gold_sense_keys":["live%2:42:08::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"live%2:42:08::","call_ids":["senseval3.d002.s082.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"live%2:42:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019025,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.942531125037931}
{"item_id":"senseval3.d002.s082.t010","gold_sense_keys":["take%2:40:08::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":34,"chosen_sense_key":"take%2:40:08::","call_ids":["senseval3.d002.s082.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":34,"predicted_sense_key":"take%2:40:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2117,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00044504999999999995,"input_uncached_usd":0.00010585,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.645354000036605}
{"item_id":"senseval3.d002.s082.t011","gold_sense_keys":["refuge%1:06:00::"],"candidates":[{"index":1,"sense_key":"refuge%1:15:00::","synset_id":"safety.n.02"},{"index":2,"sense_key":"refuge%1:07:00::","synset_id":"recourse.n.02"},{"index":3,"sense_key":"refuge%1:06:00::","synset_id":"refuge.n.03"},{"index":4,"sense_key":"refuge%1:04:00::","synset_id":"recourse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"refuge%1:06:00::","call_ids":["senseval3.d002.s082.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"refuge%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015519999999999998,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7276705410331488}
{"item_id":"senseval3.d002.s083.t001","gold_sense_keys":["distressed%5:00:01:troubled:00","distressed%5:00:02:troubled:00"],"candidates":[{"index":1,"sense_key":"distressed%5:00:00:troubled:00","synset_id":"distressed.s.01"},{"index":2,"sense_key":"distressed%3:00:04::","synset_id":"dysphoric.a.01"},{"index":3,"sense_key":"distressed%5:00:02:troubled:00","synset_id":"stressed.s.01"},{"index":4,"sense_key":"distressed%5:00:01:troubled:00","synset_id":"disquieted.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"distressed%5:00:01:troubled:00","call_ids":["senseval3.d002.s083.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"distressed%5:00:01:troubled:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015805,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7488674169871956}
{"item_id":"senseval3.d002.s084.t001","gold_sense_keys":["roll%2:38:01::","roll%2:38:10::"],"candidates":[{"index":1,"sense_key":"roll%2:38:00::","synset_id":"roll.v.01"},{"index":2,"sense_key":"roll%2:38:01::","synset_id":"wheel.v.03"},{"index":3,"sense_key":"roll%2:42:00::","synset_id":"roll.v.03"},{"index":4,"sense_key":"roll%2:35:05::","synset_id":"roll_out.v.01"},{"index":5,"sense_key":"roll%2:39:00::","synset_id":"roll.v.05"},{"index":6,"sense_key":"roll%2:35:00::","synset_id":"wind.v.03"},{"index":7,"sense_key":"roll%2:35:01::","synset_id":"roll.v.07"},{"index":8,"sense_key":"roll%2:30:10::","synset_id":"roll.v.08"},{"index":9,"sense_key":"roll%2:29:06::","synset_id":"roll.v.09"},{"index":10,"sense_key":"roll%2:40:00::","synset_id":"hustle.v.03"},{"index":11,"sense_key":"roll%2:38:10::","synset_id":"roll.v.11"},{"index":12,"sense_key":"roll%2:38:09::","synset_id":"roll.v.12"},{"index":13,"sense_key":"roll%2:38:12::","synset_id":"roll.v.13"},{"index":14,"sense_key":"roll%2:38:11::","synset_id":"roll.v.14"},{"index":15,"sense_key":"roll%2:32:00::","synset_id":"roll.v.15"},{"index":16,"sense_key":"roll%2:30:08::","synset_id":"seethe.v.04"},{"index":17,"sense_key":"roll%2:30:06::","synset_id":"roll.v.17"},{"index":18,"sense_key":"roll%2:30:07::","synset_id":"roll.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"roll%2:38:12::","call_ids":["senseval3.d002.s084.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"roll%2:38:12::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1006,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001847,"input_uncached_usd":0.000050299999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.331167500000447}
{"item_id":"senseval3.d002.s084.t002","gold_sense_keys":["town%1:15:00::"],"candidates":[{"index":1,"sense_key":"town%1:15:00::","synset_id":"town.n.01"},{"index":2,"sense_key":"town%1:14:00::","synset_id":"town.n.02"},{"index":3,"sense_key":"town%1:15:01::","synset_id":"township.n.01"},{"index":4,"sense_key":"town%1:18:00::","synset_id":"town.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"town%1:15:00::","call_ids":["senseval3.d002.s084.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"town%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018175,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1199302079621702}
{"item_id":"senseval3.d002.s086.t001","gold_sense_keys":["get%2:30:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"get%2:30:00::","call_ids":["senseval3.d002.s086.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"get%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1936,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00025679999999999995,"input_uncached_usd":0.0000968,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9189309169887565}
{"item_id":"senseval3.d002.s086.t002","gold_sense_keys":["run%2:38:00::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"run%2:38:04::","call_ids":["senseval3.d002.s086.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"run%2:38:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":2002,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00023449999999999998,"input_uncached_usd":0.0001001,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7282035830430686}
{"item_id":"senseval3.d002.s086.t003","gold_sense_keys":["door%1:06:00::","door%1:06:01::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"door%1:06:01::","call_ids":["senseval3.d002.s086.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"door%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005458,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.007269541965798}
{"item_id":"senseval3.d002.s086.t004","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval3.d002.s086.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002827,"input_uncached_usd":0.0000203,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.200663583003916}
{"item_id":"senseval3.d002.s086.t005","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d002.s086.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013069999999999998,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.211826000013389}
{"item_id":"senseval3.d002.s086.t007","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s086.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016174999999999997,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.173423999978695}
{"item_id":"senseval3.d002.s086.t008","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s086.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042105,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.051661583012901}
{"item_id":"senseval3.d002.s086.t009","gold_sense_keys":["brick%1:06:00::"],"candidates":[{"index":1,"sense_key":"brick%1:06:00::","synset_id":"brick.n.01"},{"index":2,"sense_key":"brick%1:18:00::","synset_id":"brick.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brick%1:06:00::","call_ids":["senseval3.d002.s086.t009__v1__a1","senseval3.d002.s086.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brick%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048675,"input_uncached_usd":0.00003874999999999999,"input_cached_usd":0.0,"output_usd":0.000448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.306766958965454}
{"item_id":"senseval3.d002.s086.t011","gold_sense_keys":["foot%1:08:01::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"foot%1:08:01::","call_ids":["senseval3.d002.s086.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"foot%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050525,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.661651999980677}
{"item_id":"senseval3.d002.s086.t013","gold_sense_keys":["flower%1:20:00::"],"candidates":[{"index":1,"sense_key":"flower%1:20:00::","synset_id":"flower.n.01"},{"index":2,"sense_key":"flower%1:20:02::","synset_id":"flower.n.02"},{"index":3,"sense_key":"flower%1:28:00::","synset_id":"flower.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flower%1:20:00::","call_ids":["senseval3.d002.s086.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flower%1:20:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048754999999999995,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.38939174998086}
{"item_id":"senseval3.d002.s086.t014","gold_sense_keys":["wave%2:38:00::"],"candidates":[{"index":1,"sense_key":"wave%2:32:00::","synset_id":"beckon.v.01"},{"index":2,"sense_key":"wave%2:35:00::","synset_id":"brandish.v.01"},{"index":3,"sense_key":"wave%2:38:00::","synset_id":"roll.v.11"},{"index":4,"sense_key":"wave%2:35:01::","synset_id":"curl.v.04"},{"index":5,"sense_key":"wave%2:29:00::","synset_id":"wave.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wave%2:38:00::","call_ids":["senseval3.d002.s086.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wave%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023649999999999998,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.821371124999132}
{"item_id":"senseval3.d002.s087.t000","gold_sense_keys":["amazing%5:00:00:surprising:00"],"candidates":[{"index":1,"sense_key":"amazing%5:00:00:surprising:00","synset_id":"amazing.s.01"},{"index":2,"sense_key":"amazing%5:00:00:impressive:00","synset_id":"amazing.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amazing%5:00:00:surprising:00","call_ids":["senseval3.d002.s087.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amazing%5:00:00:surprising:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00082025,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.799954375019297}
{"item_id":"senseval3.d002.s087.t001","gold_sense_keys":["do%2:36:02::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"do%2:36:02::","call_ids":["senseval3.d002.s087.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"do%2:36:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022994999999999998,"input_uncached_usd":0.00004435,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8076819999841973}
{"item_id":"senseval3.d002.s087.t002","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval3.d002.s087.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048669999999999996,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.426298125006724}
{"item_id":"senseval3.d002.s087.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["senseval3.d002.s087.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006403,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.853018291993067}
{"item_id":"senseval3.d002.s088.t000","gold_sense_keys":["look%2:29:00::","look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["senseval3.d002.s088.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022114999999999998,"input_uncached_usd":0.00003555,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.23982870799955}
{"item_id":"senseval3.d002.s088.t001","gold_sense_keys":["calm%5:00:00:composed:00"],"candidates":[{"index":1,"sense_key":"calm%5:00:00:composed:00","synset_id":"calm.s.01"},{"index":2,"sense_key":"calm%3:00:00::","synset_id":"calm.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"calm%5:00:00:composed:00","call_ids":["senseval3.d002.s088.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"calm%5:00:00:composed:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002802,"input_uncached_usd":0.0000178,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.322172500018496}
{"item_id":"senseval3.d002.s088.t002","gold_sense_keys":["surreal%5:00:00:unreal:02"],"candidates":[{"index":1,"sense_key":"surreal%5:00:00:unrealistic:00","synset_id":"phantasmagoric.s.01"},{"index":2,"sense_key":"surreal%5:00:00:unreal:02","synset_id":"dreamlike.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"surreal%5:00:00:unreal:02","call_ids":["senseval3.d002.s088.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"surreal%5:00:00:unreal:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005384000000000001,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.608981333032716}
{"item_id":"senseval3.d002.s088.t003","gold_sense_keys":["low%3:00:02::"],"candidates":[{"index":1,"sense_key":"low%3:00:02::","synset_id":"low.a.01"},{"index":2,"sense_key":"low%3:00:01::","synset_id":"low.a.02"},{"index":3,"sense_key":"low%5:00:00:soft:04","synset_id":"low.s.01"},{"index":4,"sense_key":"low%5:00:00:unrefined:01","synset_id":"low.s.02"},{"index":5,"sense_key":"low%3:00:03::","synset_id":"low.a.05"},{"index":6,"sense_key":"low%5:00:00:contemptible:00","synset_id":"abject.s.01"},{"index":7,"sense_key":"low%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":8,"sense_key":"low%5:00:00:insufficient:00","synset_id":"depleted.s.01"},{"index":9,"sense_key":"low%5:00:00:humble:00","synset_id":"broken.s.01"},{"index":10,"sense_key":"low%5:00:00:dejected:00","synset_id":"gloomy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"low%3:00:02::","call_ids":["senseval3.d002.s088.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"low%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002469,"input_uncached_usd":0.0000357,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.075631542014889}
{"item_id":"senseval3.d002.s088.t004","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["senseval3.d002.s088.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021705,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.336862583004404}
{"item_id":"senseval3.d002.s088.t005","gold_sense_keys":["confusion%1:09:00::"],"candidates":[{"index":1,"sense_key":"confusion%1:26:01::","synset_id":"confusion.n.01"},{"index":2,"sense_key":"confusion%1:09:00::","synset_id":"confusion.n.02"},{"index":3,"sense_key":"confusion%1:12:00::","synset_id":"confusion.n.03"},{"index":4,"sense_key":"confusion%1:04:01::","synset_id":"confusion.n.04"},{"index":5,"sense_key":"confusion%1:04:00::","synset_id":"confusion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confusion%1:09:00::","call_ids":["senseval3.d002.s088.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confusion%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002609,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.732568082981743}
{"item_id":"senseval3.d002.s088.t007","gold_sense_keys":["continue%2:30:10::"],"candidates":[{"index":1,"sense_key":"continue%2:42:01::","synset_id":"continue.v.01"},{"index":2,"sense_key":"continue%2:32:00::","synset_id":"continue.v.02"},{"index":3,"sense_key":"continue%2:42:00::","synset_id":"continue.v.03"},{"index":4,"sense_key":"continue%2:38:00::","synset_id":"proceed.v.02"},{"index":5,"sense_key":"continue%2:41:00::","synset_id":"retain.v.02"},{"index":6,"sense_key":"continue%2:42:12::","synset_id":"continue.v.06"},{"index":7,"sense_key":"continue%2:42:11::","synset_id":"continue.v.07"},{"index":8,"sense_key":"continue%2:42:10::","synset_id":"stay.v.04"},{"index":9,"sense_key":"continue%2:42:06::","synset_id":"cover.v.03"},{"index":10,"sense_key":"continue%2:30:10::","synset_id":"continue.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"continue%2:30:10::","call_ids":["senseval3.d002.s088.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"continue%2:30:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005822499999999999,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.828152292000595}
{"item_id":"senseval3.d002.s091.t001","gold_sense_keys":["seem%2:39:00::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d002.s091.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005429,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.84184012498008}
{"item_id":"senseval3.d002.s091.t003","gold_sense_keys":["night%1:28:00::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"night%1:28:00::","call_ids":["senseval3.d002.s091.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"night%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004950499999999999,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.48647683399031}
{"item_id":"senseval3.d002.s091.t004","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["senseval3.d002.s091.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":903,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030795,"input_uncached_usd":0.00004515,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.661875958030578}
{"item_id":"senseval3.d002.s091.t007","gold_sense_keys":["sound%1:07:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sound%1:07:00::","call_ids":["senseval3.d002.s091.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sound%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":620,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.000575,"input_uncached_usd":0.000031,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.389968624978792}
{"item_id":"senseval3.d002.s091.t008","gold_sense_keys":["light%1:06:00::","light%1:19:00::"],"candidates":[{"index":1,"sense_key":"light%1:19:00::","synset_id":"light.n.01"},{"index":2,"sense_key":"light%1:06:00::","synset_id":"light.n.02"},{"index":3,"sense_key":"light%1:09:01::","synset_id":"light.n.03"},{"index":4,"sense_key":"light%1:07:01::","synset_id":"luminosity.n.01"},{"index":5,"sense_key":"light%1:15:00::","synset_id":"light.n.05"},{"index":6,"sense_key":"light%1:26:01::","synset_id":"light.n.06"},{"index":7,"sense_key":"light%1:07:00::","synset_id":"light.n.07"},{"index":8,"sense_key":"light%1:18:00::","synset_id":"light.n.08"},{"index":9,"sense_key":"light%1:26:00::","synset_id":"light.n.09"},{"index":10,"sense_key":"light%1:09:00::","synset_id":"light.n.10"},{"index":11,"sense_key":"light%1:07:02::","synset_id":"sparkle.n.01"},{"index":12,"sense_key":"light%1:09:02::","synset_id":"light.n.12"},{"index":13,"sense_key":"light%1:16:00::","synset_id":"inner_light.n.01"},{"index":14,"sense_key":"light%1:10:00::","synset_id":"light.n.14"},{"index":15,"sense_key":"light%1:06:01::","synset_id":"lighter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"light%1:06:00::","call_ids":["senseval3.d002.s091.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"light%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00069305,"input_uncached_usd":0.000046649999999999996,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.54377495899098}
{"item_id":"senseval3.d002.s092.t000","gold_sense_keys":["friend%1:18:00::","friend%1:18:02::"],"candidates":[{"index":1,"sense_key":"friend%1:18:00::","synset_id":"friend.n.01"},{"index":2,"sense_key":"friend%1:18:03::","synset_id":"ally.n.02"},{"index":3,"sense_key":"friend%1:18:02::","synset_id":"acquaintance.n.03"},{"index":4,"sense_key":"friend%1:18:04::","synset_id":"supporter.n.01"},{"index":5,"sense_key":"friend%1:18:01::","synset_id":"friend.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"friend%1:18:00::","call_ids":["senseval3.d002.s092.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"friend%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015979999999999998,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.872674709011335}
{"item_id":"senseval3.d002.s092.t001","gold_sense_keys":["underground%5:00:00:subsurface:00"],"candidates":[{"index":1,"sense_key":"underground%5:00:00:subsurface:00","synset_id":"belowground.s.01"},{"index":2,"sense_key":"underground%5:00:00:covert:00","synset_id":"clandestine.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"underground%5:00:00:subsurface:00","call_ids":["senseval3.d002.s092.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"underground%5:00:00:subsurface:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038494999999999995,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.600406540965196}
{"item_id":"senseval3.d002.s092.t002","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s092.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008437,"input_uncached_usd":0.0000181,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.428931290982291}
{"item_id":"senseval3.d002.s092.t003","gold_sense_keys":["center%1:06:00::"],"candidates":[{"index":1,"sense_key":"center%1:15:01::","synset_id":"center.n.01"},{"index":2,"sense_key":"center%1:06:02::","synset_id":"center_field.n.01"},{"index":3,"sense_key":"center%1:06:00::","synset_id":"center.n.03"},{"index":4,"sense_key":"center%1:15:00::","synset_id":"center.n.04"},{"index":5,"sense_key":"center%1:09:00::","synset_id":"kernel.n.03"},{"index":6,"sense_key":"center%1:09:01::","synset_id":"center.n.06"},{"index":7,"sense_key":"center%1:08:00::","synset_id":"center.n.07"},{"index":8,"sense_key":"center%1:14:01::","synset_id":"center.n.08"},{"index":9,"sense_key":"center%1:18:01::","synset_id":"center.n.09"},{"index":10,"sense_key":"center%1:18:00::","synset_id":"center.n.10"},{"index":11,"sense_key":"center%1:15:02::","synset_id":"center.n.11"},{"index":12,"sense_key":"center%1:14:00::","synset_id":"center.n.12"},{"index":13,"sense_key":"center%1:18:02::","synset_id":"center.n.13"},{"index":14,"sense_key":"center%1:13:00::","synset_id":"center.n.14"},{"index":15,"sense_key":"center%1:06:01::","synset_id":"plaza.n.02"},{"index":16,"sense_key":"center%1:04:03::","synset_id":"center.n.16"},{"index":17,"sense_key":"center%1:04:02::","synset_id":"center.n.17"},{"index":18,"sense_key":"center%1:04:01::","synset_id":"center.n.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"center%1:06:00::","call_ids":["senseval3.d002.s092.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"center%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1066,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00029009999999999995,"input_uncached_usd":0.000053299999999999995,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.639504666032735}
{"item_id":"senseval3.d002.s092.t006","gold_sense_keys":["hit%2:35:05::"],"candidates":[{"index":1,"sense_key":"hit%2:35:03::","synset_id":"hit.v.01"},{"index":2,"sense_key":"hit%2:35:01::","synset_id":"hit.v.02"},{"index":3,"sense_key":"hit%2:35:00::","synset_id":"hit.v.03"},{"index":4,"sense_key":"hit%2:38:00::","synset_id":"reach.v.01"},{"index":5,"sense_key":"hit%2:35:05::","synset_id":"hit.v.05"},{"index":6,"sense_key":"hit%2:33:01::","synset_id":"shoot.v.01"},{"index":7,"sense_key":"hit%2:40:05::","synset_id":"stumble.v.03"},{"index":8,"sense_key":"hit%2:33:02::","synset_id":"score.v.01"},{"index":9,"sense_key":"hit%2:39:00::","synset_id":"hit.v.09"},{"index":10,"sense_key":"hit%2:33:00::","synset_id":"strike.v.04"},{"index":11,"sense_key":"hit%2:41:00::","synset_id":"murder.v.01"},{"index":12,"sense_key":"hit%2:38:02::","synset_id":"hit.v.12"},{"index":13,"sense_key":"hit%2:38:07::","synset_id":"reach.v.02"},{"index":14,"sense_key":"hit%2:35:02::","synset_id":"strike.v.10"},{"index":15,"sense_key":"hit%2:34:00::","synset_id":"hit.v.15"},{"index":16,"sense_key":"hit%2:33:03::","synset_id":"hit.v.16"},{"index":17,"sense_key":"hit%2:32:12::","synset_id":"hit.v.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"hit%2:38:00::","call_ids":["senseval3.d002.s092.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"hit%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1018,"cached_input_tokens":0,"output_tokens":2321,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009793,"input_uncached_usd":0.0000509,"input_cached_usd":0.0,"output_usd":0.0009283999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.095750999986194}
{"item_id":"senseval3.d002.s093.t001","gold_sense_keys":["computer%1:06:00::"],"candidates":[{"index":1,"sense_key":"computer%1:06:00::","synset_id":"computer.n.01"},{"index":2,"sense_key":"computer%1:18:00::","synset_id":"calculator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"computer%1:06:00::","call_ids":["senseval3.d002.s093.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"computer%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035759999999999996,"input_uncached_usd":0.0000184,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.191955667047296}
{"item_id":"senseval3.d002.s093.t005","gold_sense_keys":["slide%2:38:01::"],"candidates":[{"index":1,"sense_key":"slide%2:38:01::","synset_id":"skid.v.04"},{"index":2,"sense_key":"slide%2:38:00::","synset_id":"slither.v.01"},{"index":3,"sense_key":"slide%2:38:02::","synset_id":"slide.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"slide%2:38:01::","call_ids":["senseval3.d002.s093.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"slide%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036009999999999997,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.705777958035469}
{"item_id":"senseval3.d002.s093.t006","gold_sense_keys":["floor%1:06:00::"],"candidates":[{"index":1,"sense_key":"floor%1:06:00::","synset_id":"floor.n.01"},{"index":2,"sense_key":"floor%1:06:01::","synset_id":"floor.n.02"},{"index":3,"sense_key":"floor%1:10:00::","synset_id":"floor.n.03"},{"index":4,"sense_key":"floor%1:17:01::","synset_id":"floor.n.04"},{"index":5,"sense_key":"floor%1:17:00::","synset_id":"floor.n.05"},{"index":6,"sense_key":"floor%1:17:02::","synset_id":"floor.n.06"},{"index":7,"sense_key":"floor%1:14:00::","synset_id":"floor.n.07"},{"index":8,"sense_key":"floor%1:07:00::","synset_id":"floor.n.08"},{"index":9,"sense_key":"floor%1:06:03::","synset_id":"floor.n.09"},{"index":10,"sense_key":"floor%1:06:02::","synset_id":"floor.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"floor%1:06:00::","call_ids":["senseval3.d002.s093.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"floor%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021974999999999997,"input_uncached_usd":0.000034149999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.834856542001944}
{"item_id":"senseval3.d002.s094.t000","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["senseval3.d002.s094.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043349999999999997,"input_uncached_usd":0.0000175,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.068989041028544}
{"item_id":"senseval3.d002.s094.t002","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"people%1:14:03::","call_ids":["senseval3.d002.s094.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"people%1:14:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025929999999999996,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.560162041976582}
{"item_id":"senseval3.d002.s094.t003","gold_sense_keys":["realize%2:31:00::","realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"realize%2:31:01::","call_ids":["senseval3.d002.s094.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"realize%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009052,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.92166933399858}
{"item_id":"senseval3.d002.s094.t004","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s094.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017439999999999998,"input_uncached_usd":0.000039999999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.66360791597981}
{"item_id":"senseval3.d002.s094.t005","gold_sense_keys":["life%1:09:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"life%1:26:00::","call_ids":["senseval3.d002.s094.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"life%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004324,"input_uncached_usd":0.000042,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.3141915000160225}
{"item_id":"senseval3.d002.s097.t000","gold_sense_keys":["get%2:38:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"get%2:38:00::","call_ids":["senseval3.d002.s097.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"get%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1893,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00025465,"input_uncached_usd":0.00009465,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.2025249999715015}
{"item_id":"senseval3.d002.s097.t001","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s097.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000296,"input_uncached_usd":0.0000336,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.431054625019897}
{"item_id":"senseval3.d002.s097.t002","gold_sense_keys":["let%2:30:01::","let%2:32:02::","let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":3,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":4,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":5,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":6,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"let%2:32:02::","call_ids":["senseval3.d002.s097.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"let%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021224999999999998,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.326884415990207}
{"item_id":"senseval3.d002.s097.t003","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s097.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031795,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.295852165960241}
{"item_id":"senseval3.d002.s097.t004","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s097.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017109999999999998,"input_uncached_usd":0.0000367,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.859559208038263}
{"item_id":"senseval3.d002.s097.t005","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s097.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013184999999999998,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.208742333983537}
{"item_id":"senseval3.d002.s097.t006","gold_sense_keys":["sound%1:09:00::"],"candidates":[{"index":1,"sense_key":"sound%1:07:00::","synset_id":"sound.n.01"},{"index":2,"sense_key":"sound%1:09:00::","synset_id":"sound.n.02"},{"index":3,"sense_key":"sound%1:19:00::","synset_id":"sound.n.03"},{"index":4,"sense_key":"sound%1:11:00::","synset_id":"sound.n.04"},{"index":5,"sense_key":"sound%1:10:00::","synset_id":"audio.n.01"},{"index":6,"sense_key":"sound%1:10:01::","synset_id":"phone.n.02"},{"index":7,"sense_key":"sound%1:17:01::","synset_id":"strait.n.01"},{"index":8,"sense_key":"sound%1:17:00::","synset_id":"sound.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"sound%1:11:00::","call_ids":["senseval3.d002.s097.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"sound%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":2448,"reasoning_output_tokens":2432},"cost":{"currency":"USD","total_usd":0.0010079,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0009792,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.68928341701394}
{"item_id":"senseval3.d002.s097.t007","gold_sense_keys":["head%1:08:00::"],"candidates":[{"index":1,"sense_key":"head%1:08:00::","synset_id":"head.n.01"},{"index":2,"sense_key":"head%1:05:00::","synset_id":"head.n.02"},{"index":3,"sense_key":"head%1:09:00::","synset_id":"mind.n.01"},{"index":4,"sense_key":"head%1:18:00::","synset_id":"head.n.04"},{"index":5,"sense_key":"head%1:14:00::","synset_id":"head.n.05"},{"index":6,"sense_key":"head%1:19:00::","synset_id":"head.n.06"},{"index":7,"sense_key":"head%1:15:00::","synset_id":"head.n.07"},{"index":8,"sense_key":"head%1:15:01::","synset_id":"fountainhead.n.02"},{"index":9,"sense_key":"head%1:10:03::","synset_id":"head.n.09"},{"index":10,"sense_key":"head%1:26:00::","synset_id":"head.n.10"},{"index":11,"sense_key":"head%1:23:00::","synset_id":"head.n.11"},{"index":12,"sense_key":"head%1:20:00::","synset_id":"capitulum.n.01"},{"index":13,"sense_key":"head%1:18:01::","synset_id":"principal.n.02"},{"index":14,"sense_key":"head%1:18:03::","synset_id":"head.n.14"},{"index":15,"sense_key":"head%1:18:02::","synset_id":"head.n.15"},{"index":16,"sense_key":"head%1:17:02::","synset_id":"promontory.n.01"},{"index":17,"sense_key":"head%1:17:01::","synset_id":"head.n.17"},{"index":18,"sense_key":"head%1:17:00::","synset_id":"head.n.18"},{"index":19,"sense_key":"head%1:15:02::","synset_id":"forefront.n.01"},{"index":20,"sense_key":"head%1:11:01::","synset_id":"pass.n.09"},{"index":21,"sense_key":"head%1:11:00::","synset_id":"headway.n.02"},{"index":22,"sense_key":"head%1:10:01::","synset_id":"point.n.20"},{"index":23,"sense_key":"head%1:10:02::","synset_id":"question.n.02"},{"index":24,"sense_key":"head%1:10:00::","synset_id":"heading.n.01"},{"index":25,"sense_key":"head%1:08:01::","synset_id":"head.n.25"},{"index":26,"sense_key":"head%1:08:02::","synset_id":"head.n.26"},{"index":27,"sense_key":"head%1:06:05::","synset_id":"read/write_head.n.01"},{"index":28,"sense_key":"head%1:06:04::","synset_id":"head.n.28"},{"index":29,"sense_key":"head%1:06:03::","synset_id":"head.n.29"},{"index":30,"sense_key":"head%1:06:02::","synset_id":"head.n.30"},{"index":31,"sense_key":"head%1:06:00::","synset_id":"head.n.31"},{"index":32,"sense_key":"head%1:06:01::","synset_id":"drumhead.n.01"},{"index":33,"sense_key":"head%1:04:00::","synset_id":"oral_sex.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"head%1:08:00::","call_ids":["senseval3.d002.s097.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"head%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1543,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00041634999999999996,"input_uncached_usd":0.00007714999999999999,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.076990250032395}
{"item_id":"senseval3.d002.s097.t009","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"walk%2:38:03::","call_ids":["senseval3.d002.s097.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"walk%2:38:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031805,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.835490458004642}
{"item_id":"senseval3.d002.s097.t010","gold_sense_keys":["roof%1:06:00::"],"candidates":[{"index":1,"sense_key":"roof%1:06:00::","synset_id":"roof.n.01"},{"index":2,"sense_key":"roof%1:06:01::","synset_id":"roof.n.02"},{"index":3,"sense_key":"roof%1:17:00::","synset_id":"roof.n.03"},{"index":4,"sense_key":"roof%1:10:00::","synset_id":"ceiling.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"roof%1:17:00::","call_ids":["senseval3.d002.s097.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"roof%1:17:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00071845,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.601828832994215}
{"item_id":"senseval3.d002.s098.t000","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s098.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023444999999999998,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.344247833010741}
{"item_id":"senseval3.d002.s098.t001","gold_sense_keys":["car%1:06:00::"],"candidates":[{"index":1,"sense_key":"car%1:06:00::","synset_id":"car.n.01"},{"index":2,"sense_key":"car%1:06:01::","synset_id":"car.n.02"},{"index":3,"sense_key":"car%1:06:03::","synset_id":"car.n.03"},{"index":4,"sense_key":"car%1:06:02::","synset_id":"car.n.04"},{"index":5,"sense_key":"car%1:06:04::","synset_id":"cable_car.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"car%1:06:02::","call_ids":["senseval3.d002.s098.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"car%1:06:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006704,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.95160454197321}
{"item_id":"senseval3.d002.s098.t002","gold_sense_keys":["bounce%2:38:02::"],"candidates":[{"index":1,"sense_key":"bounce%2:38:00::","synset_id":"bounce.v.01"},{"index":2,"sense_key":"bounce%2:35:03::","synset_id":"bounce.v.02"},{"index":3,"sense_key":"bounce%2:38:02::","synset_id":"bounce.v.03"},{"index":4,"sense_key":"bounce%2:40:00::","synset_id":"bounce.v.04"},{"index":5,"sense_key":"bounce%2:38:01::","synset_id":"bounce.v.05"},{"index":6,"sense_key":"bounce%2:40:01::","synset_id":"bounce.v.06"},{"index":7,"sense_key":"bounce%2:35:00::","synset_id":"bounce.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bounce%2:38:02::","call_ids":["senseval3.d002.s098.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bounce%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021055,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.441826874972321}
{"item_id":"senseval3.d002.s099.t000","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d002.s099.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005219500000000001,"input_uncached_usd":0.000029149999999999997,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.280967750004493}
{"item_id":"senseval3.d002.s099.t001","gold_sense_keys":["happen%2:30:00::"],"candidates":[{"index":1,"sense_key":"happen%2:30:00::","synset_id":"happen.v.01"},{"index":2,"sense_key":"happen%2:30:02::","synset_id":"happen.v.02"},{"index":3,"sense_key":"happen%2:41:00::","synset_id":"happen.v.03"},{"index":4,"sense_key":"happen%2:30:01::","synset_id":"happen.v.04"},{"index":5,"sense_key":"happen%2:40:12::","synset_id":"find.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"happen%2:30:00::","call_ids":["senseval3.d002.s099.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"happen%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004175,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.67280879203463}
{"item_id":"senseval3.d002.s099.t002","gold_sense_keys":["scream%2:32:01::"],"candidates":[{"index":1,"sense_key":"scream%2:32:01::","synset_id":"shout.v.02"},{"index":2,"sense_key":"scream%2:32:08::","synset_id":"yell.v.02"},{"index":3,"sense_key":"scream%2:39:00::","synset_id":"scream.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"scream%2:32:08::","call_ids":["senseval3.d002.s099.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"scream%2:32:08::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033515,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3616828749654815}
{"item_id":"senseval3.d002.s099.t003","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s099.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017245,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8718226669589058}
{"item_id":"senseval3.d002.s099.t004","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s099.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031925,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.106092749978416}
{"item_id":"senseval3.d002.s100.t001","gold_sense_keys":["door%1:06:00::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"door%1:06:00::","call_ids":["senseval3.d002.s100.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"door%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046929999999999997,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.732463583000936}
{"item_id":"senseval3.d002.s100.t007","gold_sense_keys":["foot%1:23:00::"],"candidates":[{"index":1,"sense_key":"foot%1:08:01::","synset_id":"foot.n.01"},{"index":2,"sense_key":"foot%1:23:00::","synset_id":"foot.n.02"},{"index":3,"sense_key":"foot%1:15:00::","synset_id":"foot.n.03"},{"index":4,"sense_key":"foot%1:05:01::","synset_id":"animal_foot.n.01"},{"index":5,"sense_key":"foot%1:06:00::","synset_id":"foundation.n.03"},{"index":6,"sense_key":"foot%1:05:00::","synset_id":"foot.n.06"},{"index":7,"sense_key":"foot%1:04:00::","synset_id":"foot.n.07"},{"index":8,"sense_key":"foot%1:18:00::","synset_id":"foot.n.08"},{"index":9,"sense_key":"foot%1:14:00::","synset_id":"infantry.n.01"},{"index":10,"sense_key":"foot%1:10:00::","synset_id":"metrical_foot.n.01"},{"index":11,"sense_key":"foot%1:06:01::","synset_id":"foot.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foot%1:23:00::","call_ids":["senseval3.d002.s100.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037749999999999996,"input_uncached_usd":0.000038299999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.990754500031471}
{"item_id":"senseval3.d002.s100.t008","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s100.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001924,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5096785000059754}
{"item_id":"senseval3.d002.s100.t009","gold_sense_keys":["come%2:38:00::","come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"come%2:38:00::","call_ids":["senseval3.d002.s100.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"come%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1182,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002447,"input_uncached_usd":0.0000591,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.076218249974772}
{"item_id":"senseval3.d002.s100.t011","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s100.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003715999999999999,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.917082125029992}
{"item_id":"senseval3.d002.s100.t012","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s100.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037754999999999993,"input_uncached_usd":0.00003835,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.66091024997877}
{"item_id":"senseval3.d002.s100.t014","gold_sense_keys":["watch%2:39:00::","watch%2:39:02::","watch%2:41:00::"],"candidates":[{"index":1,"sense_key":"watch%2:39:00::","synset_id":"watch.v.01"},{"index":2,"sense_key":"watch%2:41:00::","synset_id":"watch.v.02"},{"index":3,"sense_key":"watch%2:39:05::","synset_id":"watch.v.03"},{"index":4,"sense_key":"watch%2:39:02::","synset_id":"watch.v.04"},{"index":5,"sense_key":"watch%2:39:01::","synset_id":"watch.v.05"},{"index":6,"sense_key":"watch%2:39:03::","synset_id":"watch.v.06"},{"index":7,"sense_key":"watch%2:32:09::","synset_id":"determine.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"watch%2:39:02::","call_ids":["senseval3.d002.s100.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"watch%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044695,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.232370915997308}
{"item_id":"senseval3.d002.s100.t015","gold_sense_keys":["outside%3:00:04::"],"candidates":[{"index":1,"sense_key":"outside%3:00:00::","synset_id":"outside.a.01"},{"index":2,"sense_key":"outside%5:00:00:extrinsic:00","synset_id":"external.s.01"},{"index":3,"sense_key":"outside%5:00:00:external:00","synset_id":"outside.s.02"},{"index":4,"sense_key":"outside%3:00:04::","synset_id":"outdoor.a.01"},{"index":5,"sense_key":"outside%5:00:00:extramural:00","synset_id":"outside.s.03"},{"index":6,"sense_key":"outside%5:00:00:exterior:00","synset_id":"outside.s.04"},{"index":7,"sense_key":"outside%5:00:00:foreign:02","synset_id":"external.s.02"},{"index":8,"sense_key":"outside%5:00:00:unlikely:00","synset_id":"outside.s.06"},{"index":9,"sense_key":"outside%5:00:00:outer:00","synset_id":"outside.s.07"},{"index":10,"sense_key":"outside%5:00:00:inaccurate:00","synset_id":"away.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"outside%3:00:04::","call_ids":["senseval3.d002.s100.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"outside%3:00:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002989,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.635454250033945}
{"item_id":"senseval3.d002.s100.t016","gold_sense_keys":["dance%2:38:02::"],"candidates":[{"index":1,"sense_key":"dance%2:38:01::","synset_id":"dance.v.01"},{"index":2,"sense_key":"dance%2:36:00::","synset_id":"dance.v.02"},{"index":3,"sense_key":"dance%2:38:02::","synset_id":"dance.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"dance%2:38:02::","call_ids":["senseval3.d002.s100.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"dance%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020774999999999998,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.967806082975585}
{"item_id":"senseval3.d002.s101.t000","gold_sense_keys":["realize%2:31:00::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["senseval3.d002.s101.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047204999999999995,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.682778583955951}
{"item_id":"senseval3.d002.s101.t001","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"go%2:38:01::","call_ids":["senseval3.d002.s101.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"go%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1711,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00057835,"input_uncached_usd":0.00008554999999999999,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.120978083985392}
{"item_id":"senseval3.d002.s101.t002","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stand%2:42:03::","call_ids":["senseval3.d002.s101.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stand%2:42:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":771,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006081500000000001,"input_uncached_usd":0.000038549999999999995,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.329783707973547}
{"item_id":"senseval3.d002.s101.t004","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s101.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004808999999999999,"input_uncached_usd":0.0000393,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.193763583025429}
{"item_id":"senseval3.d002.s101.t005","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s101.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041485000000000003,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.656663000001572}
{"item_id":"senseval3.d002.s101.t006","gold_sense_keys":["pray%2:32:00::"],"candidates":[{"index":1,"sense_key":"pray%2:32:00::","synset_id":"pray.v.01"},{"index":2,"sense_key":"pray%2:32:01::","synset_id":"beg.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"pray%2:32:00::","call_ids":["senseval3.d002.s101.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"pray%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007430000000000001,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.897981958987657}
{"item_id":"senseval3.d002.s101.t007","gold_sense_keys":["come%2:38:04::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"come%2:38:04::","call_ids":["senseval3.d002.s101.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"come%2:38:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1186,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00032169999999999996,"input_uncached_usd":0.0000593,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.330601375026163}
{"item_id":"senseval3.d002.s101.t008","gold_sense_keys":["home%1:15:04::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["senseval3.d002.s101.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027305,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.461491083027795}
{"item_id":"senseval3.d002.s101.t010","gold_sense_keys":["get%2:38:00::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"get%2:38:00::","call_ids":["senseval3.d002.s101.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"get%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1946,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002829,"input_uncached_usd":0.0000973,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0740536249941215}
{"item_id":"senseval3.d002.s102.t000","gold_sense_keys":["life%1:26:02::","life%1:28:01::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"life%1:28:01::","call_ids":["senseval3.d002.s102.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"life%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004608,"input_uncached_usd":0.0000448,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.624250916996971}
{"item_id":"senseval3.d002.s103.t000","gold_sense_keys":["see%2:39:00::","see%2:39:03::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"see%2:39:00::","call_ids":["senseval3.d002.s103.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"see%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1449,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00056525,"input_uncached_usd":0.00007245,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.757210333016701}
{"item_id":"senseval3.d002.s103.t001","gold_sense_keys":["picture%1:06:00::","picture%1:06:02::"],"candidates":[{"index":1,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":2,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":3,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":4,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":5,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":6,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":7,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":8,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":9,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":10,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"picture%1:06:02::","call_ids":["senseval3.d002.s103.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"picture%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038119999999999994,"input_uncached_usd":0.000042,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.63260062498739}
{"item_id":"senseval3.d002.s103.t002","gold_sense_keys":["begin%2:30:00::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"begin%2:30:00::","call_ids":["senseval3.d002.s103.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"begin%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":814,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006103,"input_uncached_usd":0.0000407,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.92431716696592}
{"item_id":"senseval3.d002.s103.t003","gold_sense_keys":["cry%2:29:00::"],"candidates":[{"index":1,"sense_key":"cry%2:32:01::","synset_id":"shout.v.02"},{"index":2,"sense_key":"cry%2:29:00::","synset_id":"cry.v.02"},{"index":3,"sense_key":"cry%2:32:07::","synset_id":"exclaim.v.01"},{"index":4,"sense_key":"cry%2:32:02::","synset_id":"cry.v.04"},{"index":5,"sense_key":"cry%2:34:00::","synset_id":"cry.v.05"},{"index":6,"sense_key":"cry%2:32:03::","synset_id":"cry.v.06"},{"index":7,"sense_key":"cry%2:29:02::","synset_id":"cry.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cry%2:29:00::","call_ids":["senseval3.d002.s103.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cry%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026859999999999997,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4353806669823825}
{"item_id":"senseval3.d002.s106.t000","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["senseval3.d002.s106.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005386,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.903884165978525}
{"item_id":"senseval3.d002.s106.t002","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["senseval3.d002.s106.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025929999999999996,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.12396495899884}
{"item_id":"senseval3.d002.s106.t003","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["senseval3.d002.s106.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013935,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.399197874998208}
{"item_id":"senseval3.d002.s106.t004","gold_sense_keys":["walk_around%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk_around%2:38:00::","synset_id":"perambulate.v.02"},{"index":2,"sense_key":"walk_around%2:38:01::","synset_id":"circumambulate.v.01"},{"index":3,"sense_key":"walk_around%2:41:00::","synset_id":"walk_around.v.03"},{"index":4,"sense_key":"walk_around%2:41:01::","synset_id":"walk_around.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"walk_around%2:38:00::","call_ids":["senseval3.d002.s106.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"walk_around%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002847,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.946940416994039}
{"item_id":"senseval3.d002.s106.t005","gold_sense_keys":["drink%2:34:00::"],"candidates":[{"index":1,"sense_key":"drink%2:34:00::","synset_id":"drink.v.01"},{"index":2,"sense_key":"drink%2:34:01::","synset_id":"drink.v.02"},{"index":3,"sense_key":"drink%2:34:12::","synset_id":"toast.v.02"},{"index":4,"sense_key":"drink%2:31:00::","synset_id":"drink_in.v.01"},{"index":5,"sense_key":"drink%2:34:02::","synset_id":"drink.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drink%2:34:01::","call_ids":["senseval3.d002.s106.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drink%2:34:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018454999999999999,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8064767079777084}
{"item_id":"senseval3.d002.s109.t000","gold_sense_keys":["sit_down%2:38:00::"],"candidates":[{"index":1,"sense_key":"sit_down%2:38:00::","synset_id":"sit_down.v.01"},{"index":2,"sense_key":"sit_down%2:35:00::","synset_id":"seat.v.01"},{"index":3,"sense_key":"sit_down%2:35:03::","synset_id":"sit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sit_down%2:38:00::","call_ids":["senseval3.d002.s109.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sit_down%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035634999999999996,"input_uncached_usd":0.00001715,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.487948458001483}
{"item_id":"senseval3.d002.s109.t001","gold_sense_keys":["meet%2:38:06::","meet%2:41:01::"],"candidates":[{"index":1,"sense_key":"meet%2:38:01::","synset_id":"meet.v.01"},{"index":2,"sense_key":"meet%2:41:01::","synset_id":"meet.v.02"},{"index":3,"sense_key":"meet%2:42:02::","synset_id":"converge.v.01"},{"index":4,"sense_key":"meet%2:34:00::","synset_id":"meet.v.04"},{"index":5,"sense_key":"meet%2:42:01::","synset_id":"meet.v.05"},{"index":6,"sense_key":"meet%2:41:02::","synset_id":"meet.v.06"},{"index":7,"sense_key":"meet%2:41:00::","synset_id":"meet.v.07"},{"index":8,"sense_key":"meet%2:41:03::","synset_id":"meet.v.08"},{"index":9,"sense_key":"meet%2:38:06::","synset_id":"meet.v.09"},{"index":10,"sense_key":"meet%2:33:00::","synset_id":"meet.v.10"},{"index":11,"sense_key":"meet%2:42:00::","synset_id":"meet.v.11"},{"index":12,"sense_key":"meet%2:39:00::","synset_id":"suffer.v.10"},{"index":13,"sense_key":"meet%2:35:00::","synset_id":"touch.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"meet%2:41:01::","call_ids":["senseval3.d002.s109.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"meet%2:41:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025055,"input_uncached_usd":0.00003935,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.939877249998972}
{"item_id":"senseval3.d002.s109.t002","gold_sense_keys":["new%3:00:00::","new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["senseval3.d002.s109.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035015,"input_uncached_usd":0.00003655,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3365275409887545}
{"item_id":"senseval3.d002.s109.t004","gold_sense_keys":["client%1:18:01::"],"candidates":[{"index":1,"sense_key":"client%1:18:00::","synset_id":"client.n.01"},{"index":2,"sense_key":"client%1:18:01::","synset_id":"customer.n.01"},{"index":3,"sense_key":"client%1:06:00::","synset_id":"node.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"client%1:18:01::","call_ids":["senseval3.d002.s109.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"client%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020229999999999998,"input_uncached_usd":0.0000167,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.455921458022203}
{"item_id":"senseval3.d002.s109.t005","gold_sense_keys":["couple%1:14:01::"],"candidates":[{"index":1,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":2,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":5,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"couple%1:23:00::","call_ids":["senseval3.d002.s109.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"couple%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005147,"input_uncached_usd":0.000021499999999999997,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.221482832974289}
{"item_id":"senseval3.d002.s109.t006","gold_sense_keys":["building%1:06:00::"],"candidates":[{"index":1,"sense_key":"building%1:06:00::","synset_id":"building.n.01"},{"index":2,"sense_key":"building%1:04:00::","synset_id":"construction.n.01"},{"index":3,"sense_key":"building%1:04:01::","synset_id":"construction.n.07"},{"index":4,"sense_key":"building%1:14:00::","synset_id":"building.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building%1:06:00::","call_ids":["senseval3.d002.s109.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020715,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.349135166034102}
{"item_id":"senseval3.d002.s109.t007","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval3.d002.s109.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":874,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005369,"input_uncached_usd":0.0000437,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.188038541993592}
{"item_id":"senseval3.d002.s109.t008","gold_sense_keys":["shake%2:29:00::","shake%2:38:00::","shake%2:38:01::","shake%2:38:02::"],"candidates":[{"index":1,"sense_key":"shake%2:38:00::","synset_id":"shake.v.01"},{"index":2,"sense_key":"shake%2:29:00::","synset_id":"shake.v.02"},{"index":3,"sense_key":"shake%2:38:01::","synset_id":"judder.v.01"},{"index":4,"sense_key":"shake%2:38:02::","synset_id":"rock.v.01"},{"index":5,"sense_key":"shake%2:30:00::","synset_id":"shake.v.05"},{"index":6,"sense_key":"shake%2:37:00::","synset_id":"stimulate.v.03"},{"index":7,"sense_key":"shake%2:38:04::","synset_id":"shake.v.07"},{"index":8,"sense_key":"shake%2:30:01::","synset_id":"shake.v.08"},{"index":9,"sense_key":"shake%2:32:00::","synset_id":"shake.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"shake%2:38:02::","call_ids":["senseval3.d002.s109.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"shake%2:38:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019199999999999998,"input_uncached_usd":0.000032,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0469198750215583}
{"item_id":"senseval3.d002.s110.t000","gold_sense_keys":["flimsy%5:00:01:weak:00"],"candidates":[{"index":1,"sense_key":"flimsy%5:00:01:weak:00","synset_id":"flimsy.s.01"},{"index":2,"sense_key":"flimsy%3:00:02::","synset_id":"unconvincing.a.01"},{"index":3,"sense_key":"flimsy%5:00:03:insignificant:00","synset_id":"flimsy.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flimsy%5:00:01:weak:00","call_ids":["senseval3.d002.s110.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flimsy%5:00:01:weak:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012934999999999997,"input_uncached_usd":0.000020549999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.456429500016384}
{"item_id":"senseval3.d002.s110.t001","gold_sense_keys":["structure%1:06:00::"],"candidates":[{"index":1,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":2,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":3,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":4,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":5,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"structure%1:06:00::","call_ids":["senseval3.d002.s110.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"structure%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026304999999999997,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.741590375022497}
{"item_id":"senseval3.d002.s110.t002","gold_sense_keys":["build%2:36:00::"],"candidates":[{"index":1,"sense_key":"build%2:36:00::","synset_id":"construct.v.01"},{"index":2,"sense_key":"build%2:30:00::","synset_id":"build_up.v.02"},{"index":3,"sense_key":"build%2:36:09::","synset_id":"build.v.03"},{"index":4,"sense_key":"build%2:30:02::","synset_id":"build.v.04"},{"index":5,"sense_key":"build%2:41:00::","synset_id":"build.v.05"},{"index":6,"sense_key":"build%2:36:04::","synset_id":"build.v.06"},{"index":7,"sense_key":"build%2:36:10::","synset_id":"build.v.07"},{"index":8,"sense_key":"build%2:31:03::","synset_id":"build.v.08"},{"index":9,"sense_key":"build%2:30:05::","synset_id":"build_up.v.04"},{"index":10,"sense_key":"build%2:30:10::","synset_id":"build.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"build%2:36:00::","call_ids":["senseval3.d002.s110.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"build%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001668,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.775992500013672}
{"item_id":"senseval3.d002.s110.t003","gold_sense_keys":["support%1:06:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"support%1:06:01::","call_ids":["senseval3.d002.s110.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"support%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014764999999999998,"input_uncached_usd":0.000038849999999999996,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.839130708016455}
{"item_id":"senseval3.d002.s110.t004","gold_sense_keys":["rock%2:38:00::"],"candidates":[{"index":1,"sense_key":"rock%2:38:00::","synset_id":"rock.v.01"},{"index":2,"sense_key":"rock%2:38:06::","synset_id":"rock.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rock%2:38:00::","call_ids":["senseval3.d002.s110.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rock%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":345,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025404999999999997,"input_uncached_usd":0.00001725,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.83346029202221}
{"item_id":"senseval3.d002.s111.t000","gold_sense_keys":["stop%2:42:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stop%2:42:00::","call_ids":["senseval3.d002.s111.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stop%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":791,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00068595,"input_uncached_usd":0.00003955,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.764405374997295}
{"item_id":"senseval3.d002.s111.t001","gold_sense_keys":["breathe%2:29:00::"],"candidates":[{"index":1,"sense_key":"breathe%2:29:00::","synset_id":"breathe.v.01"},{"index":2,"sense_key":"breathe%2:42:00::","synset_id":"breathe.v.02"},{"index":3,"sense_key":"breathe%2:40:00::","synset_id":"breathe.v.03"},{"index":4,"sense_key":"breathe%2:42:01::","synset_id":"breathe.v.04"},{"index":5,"sense_key":"breathe%2:32:01::","synset_id":"breathe.v.05"},{"index":6,"sense_key":"breathe%2:32:03::","synset_id":"breathe.v.06"},{"index":7,"sense_key":"breathe%2:32:00::","synset_id":"rest.v.02"},{"index":8,"sense_key":"breathe%2:30:03::","synset_id":"breathe.v.08"},{"index":9,"sense_key":"breathe%2:29:01::","synset_id":"emit.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"breathe%2:29:00::","call_ids":["senseval3.d002.s111.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"breathe%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047219999999999993,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5123721660347655}
{"item_id":"senseval3.d002.s111.t002","gold_sense_keys":["moment%1:28:01::"],"candidates":[{"index":1,"sense_key":"moment%1:28:00::","synset_id":"moment.n.01"},{"index":2,"sense_key":"moment%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"moment%1:28:02::","synset_id":"here_and_now.n.01"},{"index":4,"sense_key":"moment%1:07:00::","synset_id":"consequence.n.03"},{"index":5,"sense_key":"moment%1:19:00::","synset_id":"moment.n.05"},{"index":6,"sense_key":"moment%1:09:00::","synset_id":"moment.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"moment%1:28:01::","call_ids":["senseval3.d002.s111.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"moment%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018619999999999997,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5750418749521486}
{"item_id":"senseval3.d002.s111.t003","gold_sense_keys":["keep%2:42:07::"],"candidates":[{"index":1,"sense_key":"keep%2:42:00::","synset_id":"keep.v.01"},{"index":2,"sense_key":"keep%2:42:07::","synset_id":"continue.v.01"},{"index":3,"sense_key":"keep%2:40:00::","synset_id":"keep.v.03"},{"index":4,"sense_key":"keep%2:41:01::","synset_id":"prevent.v.02"},{"index":5,"sense_key":"keep%2:41:03::","synset_id":"observe.v.09"},{"index":6,"sense_key":"keep%2:31:00::","synset_id":"observe.v.08"},{"index":7,"sense_key":"keep%2:40:01::","synset_id":"keep.v.07"},{"index":8,"sense_key":"keep%2:32:00::","synset_id":"keep.v.08"},{"index":9,"sense_key":"keep%2:42:01::","synset_id":"keep.v.09"},{"index":10,"sense_key":"keep%2:41:02::","synset_id":"retain.v.02"},{"index":11,"sense_key":"keep%2:34:00::","synset_id":"sustain.v.04"},{"index":12,"sense_key":"keep%2:42:03::","synset_id":"keep.v.12"},{"index":13,"sense_key":"keep%2:41:05::","synset_id":"observe.v.06"},{"index":14,"sense_key":"keep%2:41:00::","synset_id":"restrain.v.01"},{"index":15,"sense_key":"keep%2:42:02::","synset_id":"keep.v.15"},{"index":16,"sense_key":"keep%2:42:12::","synset_id":"keep.v.16"},{"index":17,"sense_key":"keep%2:40:09::","synset_id":"keep_open.v.01"},{"index":18,"sense_key":"keep%2:40:13::","synset_id":"keep.v.18"},{"index":19,"sense_key":"keep%2:40:12::","synset_id":"keep.v.19"},{"index":20,"sense_key":"keep%2:40:10::","synset_id":"keep.v.20"},{"index":21,"sense_key":"keep%2:35:10::","synset_id":"keep.v.21"},{"index":22,"sense_key":"keep%2:30:10::","synset_id":"preserve.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"keep%2:42:07::","call_ids":["senseval3.d002.s111.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"keep%2:42:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1204,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00022019999999999999,"input_uncached_usd":0.0000602,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.593583457986824}
{"item_id":"senseval3.d002.s111.t004","gold_sense_keys":["come%2:30:01::"],"candidates":[{"index":1,"sense_key":"come%2:38:00::","synset_id":"come.v.01"},{"index":2,"sense_key":"come%2:38:04::","synset_id":"arrive.v.01"},{"index":3,"sense_key":"come%2:30:01::","synset_id":"come.v.03"},{"index":4,"sense_key":"come%2:30:03::","synset_id":"come.v.04"},{"index":5,"sense_key":"come%2:42:04::","synset_id":"come.v.05"},{"index":6,"sense_key":"come%2:30:02::","synset_id":"come.v.06"},{"index":7,"sense_key":"come%2:42:13::","synset_id":"issue_forth.v.01"},{"index":8,"sense_key":"come%2:42:09::","synset_id":"hail.v.02"},{"index":9,"sense_key":"come%2:42:03::","synset_id":"come.v.09"},{"index":10,"sense_key":"come%2:42:05::","synset_id":"come.v.10"},{"index":11,"sense_key":"come%2:38:03::","synset_id":"come.v.11"},{"index":12,"sense_key":"come%2:42:00::","synset_id":"fall.v.04"},{"index":13,"sense_key":"come%2:30:13::","synset_id":"come.v.13"},{"index":14,"sense_key":"come%2:42:12::","synset_id":"total.v.01"},{"index":15,"sense_key":"come%2:42:11::","synset_id":"come.v.15"},{"index":16,"sense_key":"come%2:38:02::","synset_id":"come.v.16"},{"index":17,"sense_key":"come%2:31:14::","synset_id":"occur.v.02"},{"index":18,"sense_key":"come%2:42:15::","synset_id":"derive.v.05"},{"index":19,"sense_key":"come%2:42:02::","synset_id":"do.v.04"},{"index":20,"sense_key":"come%2:39:00::","synset_id":"come.v.20"},{"index":21,"sense_key":"come%2:31:13::","synset_id":"come.v.21"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"come%2:30:01::","call_ids":["senseval3.d002.s111.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"come%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1145,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00078045,"input_uncached_usd":0.000057249999999999996,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.133239916991442}
{"item_id":"senseval3.d002.s112.t001","gold_sense_keys":["say%2:32:00::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"say%2:32:00::","call_ids":["senseval3.d002.s112.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"say%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004266,"input_uncached_usd":0.0000362,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5496503339963965}
{"item_id":"senseval3.d002.s112.t004","gold_sense_keys":["session%1:04:00::"],"candidates":[{"index":1,"sense_key":"session%1:10:00::","synset_id":"session.n.01"},{"index":2,"sense_key":"session%1:28:00::","synset_id":"school_term.n.01"},{"index":3,"sense_key":"session%1:04:00::","synset_id":"session.n.03"},{"index":4,"sense_key":"session%1:14:00::","synset_id":"seance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"session%1:04:00::","call_ids":["senseval3.d002.s112.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"session%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007725,"input_uncached_usd":0.00001965,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.6462334159878083}
{"item_id":"senseval3.d002.s116.t000","gold_sense_keys":["trouble%1:09:00::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["senseval3.d002.s116.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":786,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003379,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.0003144,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.835364374972414}
{"item_id":"senseval3.d002.s116.t005","gold_sense_keys":["spend%2:42:00::"],"candidates":[{"index":1,"sense_key":"spend%2:42:00::","synset_id":"spend.v.01"},{"index":2,"sense_key":"spend%2:40:00::","synset_id":"spend.v.02"},{"index":3,"sense_key":"spend%2:40:01::","synset_id":"spend.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"spend%2:42:00::","call_ids":["senseval3.d002.s116.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"spend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":335,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040715,"input_uncached_usd":0.00001675,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.600686166959349}
{"item_id":"senseval3.d002.s116.t007","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["senseval3.d002.s116.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046104999999999996,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.570934000017587}
{"item_id":"senseval3.d002.s116.t008","gold_sense_keys":["know%2:31:01::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["senseval3.d002.s116.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":747,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047894999999999996,"input_uncached_usd":0.00003735,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.50301337504061}
{"item_id":"senseval3.d002.s119.t002","gold_sense_keys":["start%2:30:00::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["senseval3.d002.s119.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00033145,"input_uncached_usd":0.00004345,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.423210665991064}
{"item_id":"senseval3.d002.s119.t003","gold_sense_keys":["roll%2:38:10::"],"candidates":[{"index":1,"sense_key":"roll%2:38:00::","synset_id":"roll.v.01"},{"index":2,"sense_key":"roll%2:38:01::","synset_id":"wheel.v.03"},{"index":3,"sense_key":"roll%2:42:00::","synset_id":"roll.v.03"},{"index":4,"sense_key":"roll%2:35:05::","synset_id":"roll_out.v.01"},{"index":5,"sense_key":"roll%2:39:00::","synset_id":"roll.v.05"},{"index":6,"sense_key":"roll%2:35:00::","synset_id":"wind.v.03"},{"index":7,"sense_key":"roll%2:35:01::","synset_id":"roll.v.07"},{"index":8,"sense_key":"roll%2:30:10::","synset_id":"roll.v.08"},{"index":9,"sense_key":"roll%2:29:06::","synset_id":"roll.v.09"},{"index":10,"sense_key":"roll%2:40:00::","synset_id":"hustle.v.03"},{"index":11,"sense_key":"roll%2:38:10::","synset_id":"roll.v.11"},{"index":12,"sense_key":"roll%2:38:09::","synset_id":"roll.v.12"},{"index":13,"sense_key":"roll%2:38:12::","synset_id":"roll.v.13"},{"index":14,"sense_key":"roll%2:38:11::","synset_id":"roll.v.14"},{"index":15,"sense_key":"roll%2:32:00::","synset_id":"roll.v.15"},{"index":16,"sense_key":"roll%2:30:08::","synset_id":"seethe.v.04"},{"index":17,"sense_key":"roll%2:30:06::","synset_id":"roll.v.17"},{"index":18,"sense_key":"roll%2:30:07::","synset_id":"roll.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"roll%2:42:00::","call_ids":["senseval3.d002.s119.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"roll%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":984,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038839999999999995,"input_uncached_usd":0.000049199999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.140352957998402}
{"item_id":"senseval3.d002.s119.t004","gold_sense_keys":["ocean%1:17:00::"],"candidates":[{"index":1,"sense_key":"ocean%1:17:00::","synset_id":"ocean.n.01"},{"index":2,"sense_key":"ocean%1:23:00::","synset_id":"ocean.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ocean%1:17:00::","call_ids":["senseval3.d002.s119.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ocean%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":302,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020069999999999997,"input_uncached_usd":0.0000151,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.297061917022802}
{"item_id":"senseval3.d002.s119.t005","gold_sense_keys":["wave%1:11:01::"],"candidates":[{"index":1,"sense_key":"wave%1:11:01::","synset_id":"wave.n.01"},{"index":2,"sense_key":"wave%1:04:02::","synset_id":"wave.n.02"},{"index":3,"sense_key":"wave%1:11:00::","synset_id":"wave.n.03"},{"index":4,"sense_key":"wave%1:11:02::","synset_id":"wave.n.04"},{"index":5,"sense_key":"wave%1:10:01::","synset_id":"wave.n.05"},{"index":6,"sense_key":"wave%1:08:00::","synset_id":"wave.n.06"},{"index":7,"sense_key":"wave%1:25:00::","synset_id":"wave.n.07"},{"index":8,"sense_key":"wave%1:19:00::","synset_id":"wave.n.08"},{"index":9,"sense_key":"wave%1:18:00::","synset_id":"wave.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wave%1:11:00::","call_ids":["senseval3.d002.s119.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wave%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029065,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.746286917012185}
{"item_id":"senseval3.d002.s120.t000","gold_sense_keys":["run%2:38:00::"],"candidates":[{"index":1,"sense_key":"run%2:38:00::","synset_id":"run.v.01"},{"index":2,"sense_key":"run%2:38:04::","synset_id":"scat.v.01"},{"index":3,"sense_key":"run%2:42:00::","synset_id":"run.v.03"},{"index":4,"sense_key":"run%2:41:02::","synset_id":"operate.v.01"},{"index":5,"sense_key":"run%2:42:04::","synset_id":"run.v.05"},{"index":6,"sense_key":"run%2:38:01::","synset_id":"run.v.06"},{"index":7,"sense_key":"run%2:35:00::","synset_id":"function.v.01"},{"index":8,"sense_key":"run%2:42:07::","synset_id":"range.v.01"},{"index":9,"sense_key":"run%2:33:06::","synset_id":"campaign.v.01"},{"index":10,"sense_key":"run%2:36:01::","synset_id":"play.v.18"},{"index":11,"sense_key":"run%2:38:11::","synset_id":"run.v.11"},{"index":12,"sense_key":"run%2:42:01::","synset_id":"tend.v.01"},{"index":13,"sense_key":"run%2:35:04::","synset_id":"run.v.13"},{"index":14,"sense_key":"run%2:30:03::","synset_id":"run.v.14"},{"index":15,"sense_key":"run%2:30:07::","synset_id":"run.v.15"},{"index":16,"sense_key":"run%2:42:02::","synset_id":"run.v.16"},{"index":17,"sense_key":"run%2:42:03::","synset_id":"prevail.v.03"},{"index":18,"sense_key":"run%2:42:06::","synset_id":"run.v.18"},{"index":19,"sense_key":"run%2:41:00::","synset_id":"run.v.19"},{"index":20,"sense_key":"run%2:36:02::","synset_id":"carry.v.15"},{"index":21,"sense_key":"run%2:36:00::","synset_id":"run.v.21"},{"index":22,"sense_key":"run%2:35:01::","synset_id":"guide.v.05"},{"index":23,"sense_key":"run%2:42:08::","synset_id":"run.v.23"},{"index":24,"sense_key":"run%2:41:01::","synset_id":"run.v.24"},{"index":25,"sense_key":"run%2:40:06::","synset_id":"run.v.25"},{"index":26,"sense_key":"run%2:38:10::","synset_id":"run.v.26"},{"index":27,"sense_key":"run%2:38:08::","synset_id":"run.v.27"},{"index":28,"sense_key":"run%2:38:03::","synset_id":"run.v.28"},{"index":29,"sense_key":"run%2:38:05::","synset_id":"run.v.29"},{"index":30,"sense_key":"run%2:42:05::","synset_id":"run.v.30"},{"index":31,"sense_key":"run%2:41:04::","synset_id":"run.v.31"},{"index":32,"sense_key":"run%2:38:02::","synset_id":"run.v.32"},{"index":33,"sense_key":"run%2:38:06::","synset_id":"run.v.33"},{"index":34,"sense_key":"run%2:38:13::","synset_id":"run.v.34"},{"index":35,"sense_key":"run%2:38:09::","synset_id":"ply.v.03"},{"index":36,"sense_key":"run%2:33:02::","synset_id":"hunt.v.01"},{"index":37,"sense_key":"run%2:33:01::","synset_id":"race.v.02"},{"index":38,"sense_key":"run%2:30:02::","synset_id":"move.v.13"},{"index":39,"sense_key":"run%2:30:00::","synset_id":"melt.v.01"},{"index":40,"sense_key":"run%2:30:01::","synset_id":"ladder.v.01"},{"index":41,"sense_key":"run%2:30:04::","synset_id":"run.v.41"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"run%2:38:00::","call_ids":["senseval3.d002.s120.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"run%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1952,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002832,"input_uncached_usd":0.0000976,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.531093875004444}
{"item_id":"senseval3.d002.s120.t001","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s120.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029874999999999997,"input_uncached_usd":0.000036349999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.923772417008877}
{"item_id":"senseval3.d002.s120.t002","gold_sense_keys":["get%2:35:03::"],"candidates":[{"index":1,"sense_key":"get%2:40:00::","synset_id":"get.v.01"},{"index":2,"sense_key":"get%2:30:00::","synset_id":"become.v.01"},{"index":3,"sense_key":"get%2:30:02::","synset_id":"get.v.03"},{"index":4,"sense_key":"get%2:30:03::","synset_id":"receive.v.02"},{"index":5,"sense_key":"get%2:38:00::","synset_id":"arrive.v.01"},{"index":6,"sense_key":"get%2:35:03::","synset_id":"bring.v.04"},{"index":7,"sense_key":"get%2:39:00::","synset_id":"experience.v.03"},{"index":8,"sense_key":"get%2:33:00::","synset_id":"pay_back.v.02"},{"index":9,"sense_key":"get%2:40:01::","synset_id":"have.v.17"},{"index":10,"sense_key":"get%2:32:00::","synset_id":"induce.v.02"},{"index":11,"sense_key":"get%2:35:09::","synset_id":"get.v.11"},{"index":12,"sense_key":"get%2:29:01::","synset_id":"grow.v.08"},{"index":13,"sense_key":"get%2:29:00::","synset_id":"contract.v.04"},{"index":14,"sense_key":"get%2:32:03::","synset_id":"get.v.14"},{"index":15,"sense_key":"get%2:30:01::","synset_id":"make.v.02"},{"index":16,"sense_key":"get%2:32:08::","synset_id":"drive.v.11"},{"index":17,"sense_key":"get%2:31:09::","synset_id":"catch.v.18"},{"index":18,"sense_key":"get%2:35:00::","synset_id":"catch.v.07"},{"index":19,"sense_key":"get%2:35:10::","synset_id":"get.v.19"},{"index":20,"sense_key":"get%2:31:03::","synset_id":"get.v.20"},{"index":21,"sense_key":"get%2:40:14::","synset_id":"get.v.21"},{"index":22,"sense_key":"get%2:40:13::","synset_id":"get.v.22"},{"index":23,"sense_key":"get%2:39:13::","synset_id":"catch.v.21"},{"index":24,"sense_key":"get%2:39:09::","synset_id":"catch.v.22"},{"index":25,"sense_key":"get%2:39:14::","synset_id":"get.v.25"},{"index":26,"sense_key":"get%2:38:15::","synset_id":"scram.v.01"},{"index":27,"sense_key":"get%2:38:12::","synset_id":"get.v.27"},{"index":28,"sense_key":"get%2:37:12::","synset_id":"get.v.28"},{"index":29,"sense_key":"get%2:37:11::","synset_id":"get.v.29"},{"index":30,"sense_key":"get%2:36:14::","synset_id":"catch.v.24"},{"index":31,"sense_key":"get%2:36:00::","synset_id":"draw.v.15"},{"index":32,"sense_key":"get%2:35:12::","synset_id":"get.v.32"},{"index":33,"sense_key":"get%2:31:10::","synset_id":"perplex.v.01"},{"index":34,"sense_key":"get%2:30:12::","synset_id":"get_down.v.07"},{"index":35,"sense_key":"get%2:29:11::","synset_id":"suffer.v.02"},{"index":36,"sense_key":"get%2:29:08::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"get%2:35:03::","call_ids":["senseval3.d002.s120.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"get%2:35:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1886,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0005103,"input_uncached_usd":0.0000943,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.981985874997918}
{"item_id":"senseval3.d002.s120.t003","gold_sense_keys":["next%5:00:00:succeeding:00"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"next%5:00:00:succeeding:00","call_ids":["senseval3.d002.s120.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"next%5:00:00:succeeding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048659999999999996,"input_uncached_usd":0.0000194,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.699792583996896}
{"item_id":"senseval3.d002.s120.t004","gold_sense_keys":["tremor%1:11:00::"],"candidates":[{"index":1,"sense_key":"tremor%1:04:00::","synset_id":"tremor.n.01"},{"index":2,"sense_key":"tremor%1:11:00::","synset_id":"tremor.n.02"},{"index":3,"sense_key":"tremor%1:26:00::","synset_id":"tremor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tremor%1:11:00::","call_ids":["senseval3.d002.s120.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tremor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":352,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017759999999999998,"input_uncached_usd":0.0000176,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4892350410227664}
{"item_id":"senseval3.d002.s120.t005","gold_sense_keys":["throw%2:35:00::"],"candidates":[{"index":1,"sense_key":"throw%2:35:00::","synset_id":"throw.v.01"},{"index":2,"sense_key":"throw%2:38:00::","synset_id":"throw.v.02"},{"index":3,"sense_key":"throw%2:35:07::","synset_id":"shed.v.01"},{"index":4,"sense_key":"throw%2:35:02::","synset_id":"throw.v.04"},{"index":5,"sense_key":"throw%2:32:00::","synset_id":"give.v.07"},{"index":6,"sense_key":"throw%2:35:01::","synset_id":"throw.v.06"},{"index":7,"sense_key":"throw%2:36:02::","synset_id":"project.v.10"},{"index":8,"sense_key":"throw%2:38:03::","synset_id":"throw.v.08"},{"index":9,"sense_key":"throw%2:37:00::","synset_id":"bewilder.v.02"},{"index":10,"sense_key":"throw%2:32:03::","synset_id":"hurl.v.03"},{"index":11,"sense_key":"throw%2:36:08::","synset_id":"hold.v.03"},{"index":12,"sense_key":"throw%2:36:00::","synset_id":"throw.v.12"},{"index":13,"sense_key":"throw%2:35:06::","synset_id":"throw.v.13"},{"index":14,"sense_key":"throw%2:35:08::","synset_id":"throw.v.14"},{"index":15,"sense_key":"throw%2:31:00::","synset_id":"confuse.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"throw%2:35:00::","call_ids":["senseval3.d002.s120.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"throw%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00023004999999999998,"input_uncached_usd":0.000044449999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.516141542000696}
{"item_id":"senseval3.d002.s120.t006","gold_sense_keys":["air%1:15:00::"],"candidates":[{"index":1,"sense_key":"air%1:27:00::","synset_id":"air.n.01"},{"index":2,"sense_key":"air%1:15:00::","synset_id":"air.n.02"},{"index":3,"sense_key":"air%1:07:00::","synset_id":"air.n.03"},{"index":4,"sense_key":"air%1:19:00::","synset_id":"breeze.n.01"},{"index":5,"sense_key":"air%1:15:01::","synset_id":"atmosphere.n.03"},{"index":6,"sense_key":"air%1:27:01::","synset_id":"air.n.06"},{"index":7,"sense_key":"air%1:10:01::","synset_id":"tune.n.01"},{"index":8,"sense_key":"air%1:10:02::","synset_id":"air.n.08"},{"index":9,"sense_key":"air%1:04:00::","synset_id":"air_travel.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"air%1:15:00::","call_ids":["senseval3.d002.s120.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"air%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044984999999999996,"input_uncached_usd":0.000033849999999999996,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.107879167015199}
{"item_id":"senseval3.d002.s120.t007","gold_sense_keys":["bounce%2:35:03::"],"candidates":[{"index":1,"sense_key":"bounce%2:38:00::","synset_id":"bounce.v.01"},{"index":2,"sense_key":"bounce%2:35:03::","synset_id":"bounce.v.02"},{"index":3,"sense_key":"bounce%2:38:02::","synset_id":"bounce.v.03"},{"index":4,"sense_key":"bounce%2:40:00::","synset_id":"bounce.v.04"},{"index":5,"sense_key":"bounce%2:38:01::","synset_id":"bounce.v.05"},{"index":6,"sense_key":"bounce%2:40:01::","synset_id":"bounce.v.06"},{"index":7,"sense_key":"bounce%2:35:00::","synset_id":"bounce.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bounce%2:38:02::","call_ids":["senseval3.d002.s120.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bounce%2:38:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004412,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.393486667016987}
{"item_id":"senseval3.d002.s120.t008","gold_sense_keys":["try%2:41:00::"],"candidates":[{"index":1,"sense_key":"try%2:41:00::","synset_id":"try.v.01"},{"index":2,"sense_key":"try%2:41:01::","synset_id":"test.v.01"},{"index":3,"sense_key":"try%2:41:06::","synset_id":"judge.v.05"},{"index":4,"sense_key":"try%2:34:00::","synset_id":"sample.v.01"},{"index":5,"sense_key":"try%2:41:05::","synset_id":"hear.v.03"},{"index":6,"sense_key":"try%2:37:01::","synset_id":"try.v.06"},{"index":7,"sense_key":"try%2:37:00::","synset_id":"try.v.07"},{"index":8,"sense_key":"try%2:30:00::","synset_id":"try.v.08"},{"index":9,"sense_key":"try%2:29:00::","synset_id":"try_on.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"try%2:41:00::","call_ids":["senseval3.d002.s120.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"try%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016884999999999998,"input_uncached_usd":0.00003445,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.620674333011266}
{"item_id":"senseval3.d002.s121.t000","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s121.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013394999999999998,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2747013330226764}
{"item_id":"senseval3.d002.s122.t001","gold_sense_keys":["full%3:00:00::"],"candidates":[{"index":1,"sense_key":"full%3:00:00::","synset_id":"full.a.01"},{"index":2,"sense_key":"full%5:00:01:whole:00","synset_id":"entire.s.01"},{"index":3,"sense_key":"full%5:00:00:complete:00","synset_id":"full.s.02"},{"index":4,"sense_key":"full%5:00:00:nourished:00","synset_id":"full.s.03"},{"index":5,"sense_key":"full%3:00:01::","synset_id":"full.a.05"},{"index":6,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":7,"sense_key":"full%5:00:00:high:02","synset_id":"broad.s.04"},{"index":8,"sense_key":"full%5:00:02:ample:00","synset_id":"wide.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"full%3:00:00::","call_ids":["senseval3.d002.s122.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"full%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026474999999999996,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4906534580513835}
{"item_id":"senseval3.d002.s122.t002","gold_sense_keys":["crystal%1:06:01::"],"candidates":[{"index":1,"sense_key":"crystal%1:27:00::","synset_id":"crystal.n.01"},{"index":2,"sense_key":"crystal%1:06:02::","synset_id":"crystal.n.02"},{"index":3,"sense_key":"crystal%1:17:00::","synset_id":"crystal.n.03"},{"index":4,"sense_key":"crystal%1:27:01::","synset_id":"quartz_glass.n.01"},{"index":5,"sense_key":"crystal%1:06:01::","synset_id":"crystal.n.05"},{"index":6,"sense_key":"crystal%1:06:00::","synset_id":"crystal.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"crystal%1:06:01::","call_ids":["senseval3.d002.s122.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"crystal%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020754999999999998,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.616562250012066}
{"item_id":"senseval3.d002.s123.t000","gold_sense_keys":["book%1:06:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"book%1:06:00::","call_ids":["senseval3.d002.s123.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"book%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003501,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.627776916953735}
{"item_id":"senseval3.d002.s123.t001","gold_sense_keys":["tape%1:06:01::","tape%1:06:04::"],"candidates":[{"index":1,"sense_key":"tape%1:06:00::","synset_id":"tape.n.01"},{"index":2,"sense_key":"tape%1:06:01::","synset_id":"tape.n.02"},{"index":3,"sense_key":"tape%1:15:00::","synset_id":"tape.n.03"},{"index":4,"sense_key":"tape%1:06:03::","synset_id":"tape.n.04"},{"index":5,"sense_key":"tape%1:06:04::","synset_id":"magnetic_tape.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"tape%1:06:04::","call_ids":["senseval3.d002.s123.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"tape%1:06:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":2065,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00085095,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.0008259999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.690153208968695}
{"item_id":"senseval3.d002.s123.t002","gold_sense_keys":["room%1:06:00::"],"candidates":[{"index":1,"sense_key":"room%1:06:00::","synset_id":"room.n.01"},{"index":2,"sense_key":"room%1:23:00::","synset_id":"room.n.02"},{"index":3,"sense_key":"room%1:26:00::","synset_id":"room.n.03"},{"index":4,"sense_key":"room%1:14:00::","synset_id":"room.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"room%1:06:00::","call_ids":["senseval3.d002.s123.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"room%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002049,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5970152909867465}
{"item_id":"senseval3.d002.s124.t000","gold_sense_keys":["thing%1:06:00::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"thing%1:03:00::","call_ids":["senseval3.d002.s124.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"thing%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007827999999999999,"input_uncached_usd":0.000034,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.063803084020037}
{"item_id":"senseval3.d002.s124.t001","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["senseval3.d002.s124.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037705,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.306577833951451}
{"item_id":"senseval3.d002.s124.t002","gold_sense_keys":["structure%1:06:00::"],"candidates":[{"index":1,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":2,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":3,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":4,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":5,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"structure%1:07:00::","call_ids":["senseval3.d002.s124.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"structure%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005200000000000001,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.494168166012969}
{"item_id":"senseval3.d002.s124.t003","gold_sense_keys":["fine%5:00:00:satisfactory:00"],"candidates":[{"index":1,"sense_key":"fine%5:00:00:satisfactory:00","synset_id":"all_right.s.01"},{"index":2,"sense_key":"fine%5:00:00:precise:00","synset_id":"fine.s.02"},{"index":3,"sense_key":"fine%5:00:00:thin:01","synset_id":"fine.s.03"},{"index":4,"sense_key":"fine%5:00:00:elegant:00","synset_id":"fine.s.04"},{"index":5,"sense_key":"fine%3:00:00::","synset_id":"fine.a.05"},{"index":6,"sense_key":"fine%5:00:00:pure:02","synset_id":"fine.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fine%5:00:00:satisfactory:00","call_ids":["senseval3.d002.s124.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fine%5:00:00:satisfactory:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026355,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.14775408396963}
{"item_id":"senseval3.d002.s125.t000","gold_sense_keys":["stand%2:35:00::","stand%2:42:03::"],"candidates":[{"index":1,"sense_key":"stand%2:35:00::","synset_id":"stand.v.01"},{"index":2,"sense_key":"stand%2:42:04::","synset_id":"stand.v.02"},{"index":3,"sense_key":"stand%2:42:03::","synset_id":"stand.v.03"},{"index":4,"sense_key":"stand%2:42:00::","synset_id":"stand.v.04"},{"index":5,"sense_key":"stand%2:31:00::","synset_id":"digest.v.03"},{"index":6,"sense_key":"stand%2:31:01::","synset_id":"stand.v.06"},{"index":7,"sense_key":"stand%2:42:06::","synset_id":"stand.v.07"},{"index":8,"sense_key":"stand%2:42:02::","synset_id":"stand.v.08"},{"index":9,"sense_key":"stand%2:42:01::","synset_id":"stand.v.09"},{"index":10,"sense_key":"stand%2:35:01::","synset_id":"stand.v.10"},{"index":11,"sense_key":"stand%2:33:00::","synset_id":"resist.v.04"},{"index":12,"sense_key":"stand%2:35:10::","synset_id":"stand.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stand%2:35:00::","call_ids":["senseval3.d002.s125.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stand%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004013,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.334787208004855}
{"item_id":"senseval3.d002.s125.t002","gold_sense_keys":["wait%2:42:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wait%2:42:00::","call_ids":["senseval3.d002.s125.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wait%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046399999999999995,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.814299540943466}
{"item_id":"senseval3.d002.s125.t003","gold_sense_keys":["tremor%1:11:00::"],"candidates":[{"index":1,"sense_key":"tremor%1:04:00::","synset_id":"tremor.n.01"},{"index":2,"sense_key":"tremor%1:11:00::","synset_id":"tremor.n.02"},{"index":3,"sense_key":"tremor%1:26:00::","synset_id":"tremor.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tremor%1:11:00::","call_ids":["senseval3.d002.s125.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tremor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017849999999999997,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8010290000238456}
{"item_id":"senseval3.d002.s125.t004","gold_sense_keys":["notice%2:39:00::"],"candidates":[{"index":1,"sense_key":"notice%2:39:04::","synset_id":"detect.v.01"},{"index":2,"sense_key":"notice%2:39:00::","synset_id":"notice.v.02"},{"index":3,"sense_key":"notice%2:32:00::","synset_id":"comment.v.01"},{"index":4,"sense_key":"notice%2:32:01::","synset_id":"notice.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notice%2:39:00::","call_ids":["senseval3.d002.s125.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notice%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036284999999999996,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.434284915972967}
{"item_id":"senseval3.d002.s125.t006","gold_sense_keys":["emerge%2:30:00::","emerge%2:30:02::","emerge%2:38:00::"],"candidates":[{"index":1,"sense_key":"emerge%2:30:00::","synset_id":"emerge.v.01"},{"index":2,"sense_key":"emerge%2:30:02::","synset_id":"issue.v.04"},{"index":3,"sense_key":"emerge%2:30:01::","synset_id":"emerge.v.03"},{"index":4,"sense_key":"emerge%2:38:00::","synset_id":"emerge.v.04"},{"index":5,"sense_key":"emerge%2:42:00::","synset_id":"come_forth.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emerge%2:30:02::","call_ids":["senseval3.d002.s125.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emerge%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061855,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.707096584024839}
{"item_id":"senseval3.d002.s125.t007","gold_sense_keys":["ground%1:27:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":3,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":4,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":5,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":6,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":7,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":8,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":11,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ground%1:27:00::","call_ids":["senseval3.d002.s125.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ground%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040445,"input_uncached_usd":0.00003965,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.83084091701312}
{"item_id":"senseval3.d002.s128.t000","gold_sense_keys":["amazing%5:00:00:surprising:00"],"candidates":[{"index":1,"sense_key":"amazing%5:00:00:surprising:00","synset_id":"amazing.s.01"},{"index":2,"sense_key":"amazing%5:00:00:impressive:00","synset_id":"amazing.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amazing%5:00:00:surprising:00","call_ids":["senseval3.d002.s128.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amazing%5:00:00:surprising:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001524,"input_uncached_usd":0.000018,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.963433208002243}
{"item_id":"senseval3.d002.s128.t001","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"second%1:28:01::","call_ids":["senseval3.d002.s128.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"second%1:28:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039775,"input_uncached_usd":0.00003295,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.632219458988402}
{"item_id":"senseval3.d002.s128.t002","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":3,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":4,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":5,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":6,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":7,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":8,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":9,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":10,"sense_key":"change%2:30:06::","synset_id":"change.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%2:30:01::","call_ids":["senseval3.d002.s128.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017194999999999998,"input_uncached_usd":0.000037149999999999995,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.012762958998792}
{"item_id":"senseval3.d002.s128.t003","gold_sense_keys":["life%1:26:02::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"life%1:26:02::","call_ids":["senseval3.d002.s128.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"life%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":978,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043175,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.00039119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.631432124995627}
{"item_id":"senseval3.d002.s131.t000","gold_sense_keys":["guess%2:31:00::"],"candidates":[{"index":1,"sense_key":"guess%2:31:00::","synset_id":"think.v.02"},{"index":2,"sense_key":"guess%2:32:00::","synset_id":"guess.v.02"},{"index":3,"sense_key":"guess%2:31:01::","synset_id":"estimate.v.01"},{"index":4,"sense_key":"guess%2:31:02::","synset_id":"guess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guess%2:31:00::","call_ids":["senseval3.d002.s131.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018165,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.060754749982152}
{"item_id":"senseval3.d002.s131.t001","gold_sense_keys":["live%2:42:04::","live%2:42:06::"],"candidates":[{"index":1,"sense_key":"live%2:42:08::","synset_id":"populate.v.01"},{"index":2,"sense_key":"live%2:42:06::","synset_id":"live.v.02"},{"index":3,"sense_key":"live%2:42:04::","synset_id":"survive.v.01"},{"index":4,"sense_key":"live%2:42:01::","synset_id":"exist.v.02"},{"index":5,"sense_key":"live%2:42:00::","synset_id":"be.v.11"},{"index":6,"sense_key":"live%2:31:00::","synset_id":"know.v.05"},{"index":7,"sense_key":"live%2:42:07::","synset_id":"live.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"live%2:42:04::","call_ids":["senseval3.d002.s131.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"live%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034179999999999996,"input_uncached_usd":0.000028199999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.569314999971539}
{"item_id":"senseval3.d002.s131.t002","gold_sense_keys":["wait%2:31:00::"],"candidates":[{"index":1,"sense_key":"wait%2:42:00::","synset_id":"wait.v.01"},{"index":2,"sense_key":"wait%2:42:01::","synset_id":"wait.v.02"},{"index":3,"sense_key":"wait%2:31:00::","synset_id":"expect.v.03"},{"index":4,"sense_key":"wait%2:41:00::","synset_id":"wait.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"wait%2:31:00::","call_ids":["senseval3.d002.s131.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"wait%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007705,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.4381232500309125}
{"item_id":"senseval3.d002.s132.t000","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["senseval3.d002.s132.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037639999999999993,"input_uncached_usd":0.000037199999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.568648750020657}
{"item_id":"senseval3.d002.s132.t001","gold_sense_keys":["accept%2:31:00::"],"candidates":[{"index":1,"sense_key":"accept%2:31:00::","synset_id":"accept.v.01"},{"index":2,"sense_key":"accept%2:40:00::","synset_id":"accept.v.02"},{"index":3,"sense_key":"accept%2:32:00::","synset_id":"accept.v.03"},{"index":4,"sense_key":"accept%2:31:01::","synset_id":"accept.v.04"},{"index":5,"sense_key":"accept%2:40:03::","synset_id":"accept.v.05"},{"index":6,"sense_key":"accept%2:40:01::","synset_id":"bear.v.06"},{"index":7,"sense_key":"accept%2:31:03::","synset_id":"accept.v.07"},{"index":8,"sense_key":"accept%2:42:00::","synset_id":"accept.v.08"},{"index":9,"sense_key":"accept%2:40:07::","synset_id":"accept.v.09"},{"index":10,"sense_key":"accept%2:40:12::","synset_id":"take.v.20"},{"index":11,"sense_key":"accept%2:31:08::","synset_id":"accept.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"accept%2:31:00::","call_ids":["senseval3.d002.s132.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"accept%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007326500000000001,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.989255124994088}
{"item_id":"senseval3.d002.s132.t002","gold_sense_keys":["take%2:40:06::"],"candidates":[{"index":1,"sense_key":"take%2:41:04::","synset_id":"take.v.01"},{"index":2,"sense_key":"take%2:40:06::","synset_id":"take.v.02"},{"index":3,"sense_key":"take%2:38:09::","synset_id":"lead.v.01"},{"index":4,"sense_key":"take%2:35:00::","synset_id":"take.v.04"},{"index":5,"sense_key":"take%2:30:01::","synset_id":"assume.v.03"},{"index":6,"sense_key":"take%2:31:07::","synset_id":"take.v.06"},{"index":7,"sense_key":"take%2:38:10::","synset_id":"bring.v.01"},{"index":8,"sense_key":"take%2:40:00::","synset_id":"take.v.08"},{"index":9,"sense_key":"take%2:38:02::","synset_id":"take.v.09"},{"index":10,"sense_key":"take%2:31:01::","synset_id":"choose.v.01"},{"index":11,"sense_key":"take%2:40:05::","synset_id":"accept.v.02"},{"index":12,"sense_key":"take%2:41:00::","synset_id":"fill.v.04"},{"index":13,"sense_key":"take%2:31:03::","synset_id":"consider.v.03"},{"index":14,"sense_key":"take%2:42:00::","synset_id":"necessitate.v.01"},{"index":15,"sense_key":"take%2:39:00::","synset_id":"take.v.15"},{"index":16,"sense_key":"take%2:32:02::","synset_id":"film.v.01"},{"index":17,"sense_key":"take%2:30:00::","synset_id":"remove.v.01"},{"index":18,"sense_key":"take%2:34:00::","synset_id":"consume.v.02"},{"index":19,"sense_key":"take%2:31:09::","synset_id":"take.v.19"},{"index":20,"sense_key":"take%2:40:15::","synset_id":"take.v.20"},{"index":21,"sense_key":"take%2:40:01::","synset_id":"take.v.21"},{"index":22,"sense_key":"take%2:38:00::","synset_id":"assume.v.05"},{"index":23,"sense_key":"take%2:40:07::","synset_id":"accept.v.05"},{"index":24,"sense_key":"take%2:30:08::","synset_id":"take.v.24"},{"index":25,"sense_key":"take%2:31:02::","synset_id":"learn.v.04"},{"index":26,"sense_key":"take%2:32:00::","synset_id":"claim.v.05"},{"index":27,"sense_key":"take%2:38:05::","synset_id":"take.v.27"},{"index":28,"sense_key":"take%2:33:09::","synset_id":"aim.v.01"},{"index":29,"sense_key":"take%2:30:09::","synset_id":"take.v.29"},{"index":30,"sense_key":"take%2:42:10::","synset_id":"carry.v.02"},{"index":31,"sense_key":"take%2:40:03::","synset_id":"lease.v.04"},{"index":32,"sense_key":"take%2:40:04::","synset_id":"subscribe.v.05"},{"index":33,"sense_key":"take%2:40:02::","synset_id":"take.v.33"},{"index":34,"sense_key":"take%2:40:08::","synset_id":"take.v.34"},{"index":35,"sense_key":"take%2:35:14::","synset_id":"take.v.35"},{"index":36,"sense_key":"take%2:32:09::","synset_id":"claim.v.04"},{"index":37,"sense_key":"take%2:42:03::","synset_id":"accept.v.08"},{"index":38,"sense_key":"take%2:42:15::","synset_id":"contain.v.05"},{"index":39,"sense_key":"take%2:41:13::","synset_id":"take.v.39"},{"index":40,"sense_key":"take%2:38:11::","synset_id":"drive.v.16"},{"index":41,"sense_key":"take%2:33:08::","synset_id":"take.v.41"},{"index":42,"sense_key":"take%2:29:08::","synset_id":"contract.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"take%2:40:06::","call_ids":["senseval3.d002.s132.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"take%2:40:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2080,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00031519999999999996,"input_uncached_usd":0.000104,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8793540420010686}
{"item_id":"senseval3.d002.s132.t003","gold_sense_keys":["second%1:28:00::"],"candidates":[{"index":1,"sense_key":"second%1:28:00::","synset_id":"second.n.01"},{"index":2,"sense_key":"second%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"second%1:04:00::","synset_id":"second_base.n.02"},{"index":4,"sense_key":"second%1:28:02::","synset_id":"moment.n.01"},{"index":5,"sense_key":"second%1:24:00::","synset_id":"second.n.05"},{"index":6,"sense_key":"second%1:23:00::","synset_id":"second.n.06"},{"index":7,"sense_key":"second%1:18:00::","synset_id":"second.n.07"},{"index":8,"sense_key":"second%1:10:00::","synset_id":"second.n.08"},{"index":9,"sense_key":"second%1:06:00::","synset_id":"second_gear.n.01"},{"index":10,"sense_key":"second%1:06:01::","synset_id":"irregular.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%1:28:00::","call_ids":["senseval3.d002.s132.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037069999999999996,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.418166875024326}
{"item_id":"senseval3.d002.s133.t000","gold_sense_keys":["wonder%2:31:00::"],"candidates":[{"index":1,"sense_key":"wonder%2:31:00::","synset_id":"wonder.v.01"},{"index":2,"sense_key":"wonder%2:32:01::","synset_id":"wonder.v.02"},{"index":3,"sense_key":"wonder%2:32:12::","synset_id":"wonder.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wonder%2:31:00::","call_ids":["senseval3.d002.s133.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wonder%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":345,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038245,"input_uncached_usd":0.00001725,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.956783334026113}
{"item_id":"senseval3.d002.s133.t001","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["senseval3.d002.s133.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":978,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041145,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.00039119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.445715582987759}
{"item_id":"senseval3.d002.s133.t002","gold_sense_keys":["relax%2:29:00::"],"candidates":[{"index":1,"sense_key":"relax%2:29:00::","synset_id":"relax.v.01"},{"index":2,"sense_key":"relax%2:30:01::","synset_id":"relax.v.02"},{"index":3,"sense_key":"relax%2:30:02::","synset_id":"loosen.v.07"},{"index":4,"sense_key":"relax%2:29:01::","synset_id":"relax.v.04"},{"index":5,"sense_key":"relax%2:41:02::","synset_id":"relax.v.05"},{"index":6,"sense_key":"relax%2:41:01::","synset_id":"relax.v.06"},{"index":7,"sense_key":"relax%2:41:00::","synset_id":"relax.v.07"},{"index":8,"sense_key":"relax%2:30:05::","synset_id":"slack.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relax%2:29:00::","call_ids":["senseval3.d002.s133.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relax%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034419999999999996,"input_uncached_usd":0.0000302,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2349275419837795}
{"item_id":"senseval3.d002.s136.t000","gold_sense_keys":["flesh%1:08:01::"],"candidates":[{"index":1,"sense_key":"flesh%1:08:02::","synset_id":"flesh.n.01"},{"index":2,"sense_key":"flesh%1:08:01::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"flesh%1:20:00::","synset_id":"pulp.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flesh%1:08:01::","call_ids":["senseval3.d002.s136.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flesh%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030485,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.061074583965819}
{"item_id":"senseval3.d002.s136.t001","gold_sense_keys":["go%2:30:04::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"go%2:30:04::","call_ids":["senseval3.d002.s136.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"go%2:30:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1617,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00039444999999999997,"input_uncached_usd":0.00008085,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.29225908301305}
{"item_id":"senseval3.d002.s136.t003","gold_sense_keys":["flight%1:04:01::"],"candidates":[{"index":1,"sense_key":"flight%1:14:01::","synset_id":"flight.n.01"},{"index":2,"sense_key":"flight%1:04:00::","synset_id":"flight.n.02"},{"index":3,"sense_key":"flight%1:06:00::","synset_id":"flight.n.03"},{"index":4,"sense_key":"flight%1:04:01::","synset_id":"escape.n.01"},{"index":5,"sense_key":"flight%1:14:00::","synset_id":"flight.n.05"},{"index":6,"sense_key":"flight%1:09:00::","synset_id":"flight.n.06"},{"index":7,"sense_key":"flight%1:19:00::","synset_id":"trajectory.n.01"},{"index":8,"sense_key":"flight%1:14:02::","synset_id":"flight.n.08"},{"index":9,"sense_key":"flight%1:04:02::","synset_id":"flight.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"flight%1:04:01::","call_ids":["senseval3.d002.s136.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"flight%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031345,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.750709582993295}
{"item_id":"senseval3.d002.s136.t004","gold_sense_keys":["fight%1:04:02::"],"candidates":[{"index":1,"sense_key":"fight%1:04:01::","synset_id":"battle.n.01"},{"index":2,"sense_key":"fight%1:04:02::","synset_id":"fight.n.02"},{"index":3,"sense_key":"fight%1:07:00::","synset_id":"competitiveness.n.01"},{"index":4,"sense_key":"fight%1:10:00::","synset_id":"fight.n.04"},{"index":5,"sense_key":"fight%1:04:03::","synset_id":"fight.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fight%1:04:02::","call_ids":["senseval3.d002.s136.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fight%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036104999999999997,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.799799708009232}
{"item_id":"senseval3.d002.s137.t000","gold_sense_keys":["nausea%1:26:00::"],"candidates":[{"index":1,"sense_key":"nausea%1:26:00::","synset_id":"nausea.n.01"},{"index":2,"sense_key":"nausea%1:12:00::","synset_id":"nausea.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nausea%1:26:00::","call_ids":["senseval3.d002.s137.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nausea%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":261,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00024985,"input_uncached_usd":0.000013049999999999999,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.37520695803687}
{"item_id":"senseval3.d002.s137.t001","gold_sense_keys":["seem%2:39:01::"],"candidates":[{"index":1,"sense_key":"seem%2:39:00::","synset_id":"look.v.02"},{"index":2,"sense_key":"seem%2:39:01::","synset_id":"appear.v.04"},{"index":3,"sense_key":"seem%2:42:00::","synset_id":"seem.v.03"},{"index":4,"sense_key":"seem%2:39:02::","synset_id":"seem.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seem%2:39:01::","call_ids":["senseval3.d002.s137.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seem%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035864999999999996,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.6888804999762215}
{"item_id":"senseval3.d002.s137.t002","gold_sense_keys":["commonplace%5:00:02:ordinary:00"],"candidates":[{"index":1,"sense_key":"commonplace%5:00:02:ordinary:00","synset_id":"commonplace.s.01"},{"index":2,"sense_key":"commonplace%5:00:00:unexciting:00","synset_id":"commonplace.s.02"},{"index":3,"sense_key":"commonplace%5:00:00:unoriginal:00","synset_id":"banal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"commonplace%5:00:02:ordinary:00","call_ids":["senseval3.d002.s137.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"commonplace%5:00:02:ordinary:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":378,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006141,"input_uncached_usd":0.0000189,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.1047233340214}
{"item_id":"senseval3.d002.s137.t003","gold_sense_keys":["symptom%1:26:00::"],"candidates":[{"index":1,"sense_key":"symptom%1:26:00::","synset_id":"symptom.n.01"},{"index":2,"sense_key":"symptom%1:10:00::","synset_id":"symptom.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"symptom%1:26:00::","call_ids":["senseval3.d002.s137.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"symptom%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":277,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006346500000000001,"input_uncached_usd":0.00001385,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.07560600002762}
{"item_id":"senseval3.d002.s139.t000","gold_sense_keys":["walk%2:38:00::"],"candidates":[{"index":1,"sense_key":"walk%2:38:00::","synset_id":"walk.v.01"},{"index":2,"sense_key":"walk%2:38:01::","synset_id":"walk.v.02"},{"index":3,"sense_key":"walk%2:33:00::","synset_id":"walk.v.03"},{"index":4,"sense_key":"walk%2:38:03::","synset_id":"walk.v.04"},{"index":5,"sense_key":"walk%2:33:01::","synset_id":"walk.v.05"},{"index":6,"sense_key":"walk%2:41:00::","synset_id":"walk.v.06"},{"index":7,"sense_key":"walk%2:41:01::","synset_id":"walk.v.07"},{"index":8,"sense_key":"walk%2:38:02::","synset_id":"walk.v.08"},{"index":9,"sense_key":"walk%2:38:05::","synset_id":"walk.v.09"},{"index":10,"sense_key":"walk%2:38:04::","synset_id":"walk.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"walk%2:38:03::","call_ids":["senseval3.d002.s139.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"walk%2:38:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002917,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.093512540974189}
{"item_id":"senseval3.d002.s139.t002","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["senseval3.d002.s139.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002612,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.325378499983344}
{"item_id":"senseval3.d002.s139.t003","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"},{"index":2,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":3,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":4,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morning%1:28:00::","call_ids":["senseval3.d002.s139.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":412,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025739999999999997,"input_uncached_usd":0.0000206,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.490432040998712}
{"item_id":"senseval3.d002.s140.t000","gold_sense_keys":["next%5:00:00:close:01"],"candidates":[{"index":1,"sense_key":"next%5:00:00:succeeding:00","synset_id":"following.s.02"},{"index":2,"sense_key":"next%5:00:00:close:01","synset_id":"adjacent.s.01"},{"index":3,"sense_key":"next%5:00:00:incoming:00","synset_id":"future.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"next%5:00:00:close:01","call_ids":["senseval3.d002.s140.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"next%5:00:00:close:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017879999999999998,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.10146616701968}
{"item_id":"senseval3.d002.s140.t004","gold_sense_keys":["blue%5:00:00:chromatic:00"],"candidates":[{"index":1,"sense_key":"blue%5:00:00:chromatic:00","synset_id":"blue.s.01"},{"index":2,"sense_key":"blue%5:00:00:northern:02","synset_id":"blue.s.02"},{"index":3,"sense_key":"blue%5:00:00:dejected:00","synset_id":"gloomy.s.02"},{"index":4,"sense_key":"blue%5:00:02:dirty:02","synset_id":"blasphemous.s.02"},{"index":5,"sense_key":"blue%5:00:00:sexy:00","synset_id":"blue.s.05"},{"index":6,"sense_key":"blue%5:00:00:noble:02","synset_id":"aristocratic.s.01"},{"index":7,"sense_key":"blue%5:00:00:nonindulgent:00","synset_id":"blue.s.07"},{"index":8,"sense_key":"blue%5:00:00:depressing:00","synset_id":"blue.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blue%5:00:00:chromatic:00","call_ids":["senseval3.d002.s140.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blue%5:00:00:chromatic:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002717,"input_uncached_usd":0.0000349,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.5016672919737175}
{"item_id":"senseval3.d002.s140.t006","gold_sense_keys":["sit_up%2:38:01::"],"candidates":[{"index":1,"sense_key":"sit_up%2:29:00::","synset_id":"stay_up.v.01"},{"index":2,"sense_key":"sit_up%2:38:01::","synset_id":"sit_up.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sit_up%2:38:01::","call_ids":["senseval3.d002.s140.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sit_up%2:38:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038114999999999997,"input_uncached_usd":0.000016349999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.530305749969557}
{"item_id":"senseval3.d002.s140.t007","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d002.s140.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024734999999999997,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.171586957992986}
{"item_id":"senseval3.d002.s140.t009","gold_sense_keys":["woman%1:18:00::"],"candidates":[{"index":1,"sense_key":"woman%1:18:00::","synset_id":"woman.n.01"},{"index":2,"sense_key":"woman%1:18:01::","synset_id":"woman.n.02"},{"index":3,"sense_key":"woman%1:18:02::","synset_id":"charwoman.n.01"},{"index":4,"sense_key":"woman%1:14:00::","synset_id":"womanhood.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woman%1:18:00::","call_ids":["senseval3.d002.s140.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woman%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020754999999999998,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.434230917016976}
{"item_id":"senseval3.d002.s140.t010","gold_sense_keys":["smile%2:29:00::"],"candidates":[{"index":1,"sense_key":"smile%2:29:00::","synset_id":"smile.v.01"},{"index":2,"sense_key":"smile%2:32:00::","synset_id":"smile.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"smile%2:29:00::","call_ids":["senseval3.d002.s140.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"smile%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":301,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00055905,"input_uncached_usd":0.00001505,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.806421042012516}
{"item_id":"senseval3.d002.s140.t011","gold_sense_keys":["say%2:32:15::"],"candidates":[{"index":1,"sense_key":"say%2:32:00::","synset_id":"state.v.01"},{"index":2,"sense_key":"say%2:32:01::","synset_id":"allege.v.01"},{"index":3,"sense_key":"say%2:32:03::","synset_id":"suppose.v.01"},{"index":4,"sense_key":"say%2:42:00::","synset_id":"read.v.02"},{"index":5,"sense_key":"say%2:32:04::","synset_id":"order.v.01"},{"index":6,"sense_key":"say%2:32:02::","synset_id":"pronounce.v.01"},{"index":7,"sense_key":"say%2:32:07::","synset_id":"say.v.07"},{"index":8,"sense_key":"say%2:32:15::","synset_id":"say.v.08"},{"index":9,"sense_key":"say%2:32:13::","synset_id":"say.v.09"},{"index":10,"sense_key":"say%2:32:08::","synset_id":"say.v.10"},{"index":11,"sense_key":"say%2:32:09::","synset_id":"say.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"say%2:32:15::","call_ids":["senseval3.d002.s140.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"say%2:32:15::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042655,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.368675667035859}
{"item_id":"senseval3.d002.s140.t012","gold_sense_keys":["great%5:00:00:good:01"],"candidates":[{"index":1,"sense_key":"great%5:00:01:large:00","synset_id":"great.s.01"},{"index":2,"sense_key":"great%5:00:01:important:00","synset_id":"great.s.02"},{"index":3,"sense_key":"great%5:00:01:extraordinary:00","synset_id":"great.s.03"},{"index":4,"sense_key":"great%5:00:00:good:01","synset_id":"bang-up.s.01"},{"index":5,"sense_key":"great%5:00:00:uppercase:00","synset_id":"capital.s.03"},{"index":6,"sense_key":"great%5:00:00:pregnant:00","synset_id":"big.s.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"great%5:00:00:good:01","call_ids":["senseval3.d002.s140.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"great%5:00:00:good:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018649999999999998,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.214489082980435}
{"item_id":"senseval3.d002.s140.t013","gold_sense_keys":["alive%3:00:01::"],"candidates":[{"index":1,"sense_key":"alive%3:00:01::","synset_id":"alive.a.01"},{"index":2,"sense_key":"alive%5:00:00:lively:00","synset_id":"alive.s.01"},{"index":3,"sense_key":"alive%3:00:04::","synset_id":"animated.a.01"},{"index":4,"sense_key":"alive%5:00:00:sensitive:02","synset_id":"alive.s.02"},{"index":5,"sense_key":"alive%5:00:00:existent:00","synset_id":"active.s.03"},{"index":6,"sense_key":"alive%5:00:00:aware:00","synset_id":"alert.s.02"},{"index":7,"sense_key":"alive%5:00:00:active:05","synset_id":"alive.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"alive%3:00:01::","call_ids":["senseval3.d002.s140.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"alive%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036539999999999994,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.684586332994513}
{"item_id":"senseval3.d002.s141.t000","gold_sense_keys":["agree%2:32:00::"],"candidates":[{"index":1,"sense_key":"agree%2:32:00::","synset_id":"agree.v.01"},{"index":2,"sense_key":"agree%2:32:01::","synset_id":"agree.v.02"},{"index":3,"sense_key":"agree%2:42:00::","synset_id":"match.v.01"},{"index":4,"sense_key":"agree%2:42:01::","synset_id":"harmonize.v.01"},{"index":5,"sense_key":"agree%2:42:03::","synset_id":"agree.v.05"},{"index":6,"sense_key":"agree%2:40:02::","synset_id":"agree.v.06"},{"index":7,"sense_key":"agree%2:32:04::","synset_id":"agree.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"agree%2:32:00::","call_ids":["senseval3.d002.s141.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"agree%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002902,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.5960632910137065}
{"item_id":"semeval2013.d000.s000.t000","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00027919999999999996,"input_uncached_usd":0.0000168,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.483831957972143}
{"item_id":"semeval2013.d000.s000.t001","gold_sense_keys":["plan%1:09:00::"],"candidates":[{"index":1,"sense_key":"plan%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"plan%1:09:01::","synset_id":"design.n.02"},{"index":3,"sense_key":"plan%1:06:00::","synset_id":"plan.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"plan%1:09:00::","call_ids":["semeval2013.d000.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"plan%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012654999999999999,"input_uncached_usd":0.000017749999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.079386874975171}
{"item_id":"semeval2013.d000.s000.t002","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":381,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028145,"input_uncached_usd":0.00001905,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.242548250011168}
{"item_id":"semeval2013.d000.s001.t000","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"climate%1:26:01::","call_ids":["semeval2013.d000.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"climate%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00084305,"input_uncached_usd":0.00001745,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.793818833946716}
{"item_id":"semeval2013.d000.s001.t001","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conference%1:10:00::","call_ids":["semeval2013.d000.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conference%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":351,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038235,"input_uncached_usd":0.00001755,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.177690624957904}
{"item_id":"semeval2013.d000.s001.t004","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":2,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"},{"index":3,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":4,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":5,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":6,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":7,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2013.d000.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016115,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7754949160153046}
{"item_id":"semeval2013.d000.s001.t007","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2013.d000.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012759999999999998,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.050705957983155}
{"item_id":"semeval2013.d000.s001.t008","gold_sense_keys":["reduction%1:04:00::"],"candidates":[{"index":1,"sense_key":"reduction%1:04:00::","synset_id":"decrease.n.04"},{"index":2,"sense_key":"reduction%1:22:00::","synset_id":"reduction.n.02"},{"index":3,"sense_key":"reduction%1:04:02::","synset_id":"reduction.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reduction%1:04:00::","call_ids":["semeval2013.d000.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reduction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017815,"input_uncached_usd":0.00001815,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2755752920056693}
{"item_id":"semeval2013.d000.s001.t009","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"year%1:28:00::","call_ids":["semeval2013.d000.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"year%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000259,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.204329250031151}
{"item_id":"semeval2013.d000.s001.t010","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d000.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020564999999999998,"input_uncached_usd":0.00002005,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9296480410266668}
{"item_id":"semeval2013.d000.s001.t011","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":2,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":3,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":4,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":5,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":6,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":7,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d000.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016184999999999998,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.438208624953404}
{"item_id":"semeval2013.d000.s002.t000","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028619999999999996,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.619608625012916}
{"item_id":"semeval2013.d000.s002.t001","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2013.d000.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020674999999999998,"input_uncached_usd":0.00002115,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.115387458004989}
{"item_id":"semeval2013.d000.s002.t002","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001681,"input_uncached_usd":0.0000337,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.248726917023305}
{"item_id":"semeval2013.d000.s002.t004","gold_sense_keys":["obstacle%1:09:00::"],"candidates":[{"index":1,"sense_key":"obstacle%1:09:00::","synset_id":"obstacle.n.01"},{"index":2,"sense_key":"obstacle%1:06:00::","synset_id":"obstacle.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obstacle%1:09:00::","call_ids":["semeval2013.d000.s002.t004__v1__a1","semeval2013.d000.s002.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obstacle%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":2208,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.00092355,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.0008832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.348491000011563}
{"item_id":"semeval2013.d000.s002.t005","gold_sense_keys":["path%1:04:00::"],"candidates":[{"index":1,"sense_key":"path%1:04:00::","synset_id":"way.n.05"},{"index":2,"sense_key":"path%1:06:00::","synset_id":"path.n.02"},{"index":3,"sense_key":"path%1:15:00::","synset_id":"path.n.03"},{"index":4,"sense_key":"path%1:17:00::","synset_id":"path.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"path%1:04:00::","call_ids":["semeval2013.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"path%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036034999999999995,"input_uncached_usd":0.00002115,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4386970839696005}
{"item_id":"semeval2013.d000.s003.t000","gold_sense_keys":["advance%1:11:01::"],"candidates":[{"index":1,"sense_key":"advance%1:11:00::","synset_id":"progress.n.03"},{"index":2,"sense_key":"advance%1:11:01::","synset_id":"improvement.n.01"},{"index":3,"sense_key":"advance%1:10:00::","synset_id":"overture.n.03"},{"index":4,"sense_key":"advance%1:04:00::","synset_id":"progress.n.02"},{"index":5,"sense_key":"advance%1:21:00::","synset_id":"advance.n.05"},{"index":6,"sense_key":"advance%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"advance%1:04:00::","call_ids":["semeval2013.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"advance%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049435,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.711267917009536}
{"item_id":"semeval2013.d000.s003.t001","gold_sense_keys":["negotiation%1:10:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"negotiation%1:04:00::","call_ids":["semeval2013.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"negotiation%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00094815,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.181648625002708}
{"item_id":"semeval2013.d000.s003.t002","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["semeval2013.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058015,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.061170917004347}
{"item_id":"semeval2013.d000.s003.t004","gold_sense_keys":["leader%1:18:00::"],"candidates":[{"index":1,"sense_key":"leader%1:18:00::","synset_id":"leader.n.01"},{"index":2,"sense_key":"leader%1:06:00::","synset_id":"drawing_card.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"leader%1:18:00::","call_ids":["semeval2013.d000.s003.t004__v1__a1","semeval2013.d000.s003.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"leader%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041175,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.955911208991893}
{"item_id":"semeval2013.d000.s003.t006","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028375,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.821760999970138}
{"item_id":"semeval2013.d000.s004.t001","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"text%1:10:01::","call_ids":["semeval2013.d000.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"text%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003671999999999999,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.689400666975416}
{"item_id":"semeval2013.d000.s004.t002","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"money%1:21:00::","call_ids":["semeval2013.d000.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"money%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038845,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.56302958301967}
{"item_id":"semeval2013.d000.s004.t003","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039539999999999996,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.052012583997566}
{"item_id":"semeval2013.d000.s004.t004","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d000.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002599,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.737507082987577}
{"item_id":"semeval2013.d000.s004.t006","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":2,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":3,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":4,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":5,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":6,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":7,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d000.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029595,"input_uncached_usd":0.00003315,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.248850250034593}
{"item_id":"semeval2013.d000.s005.t000","gold_sense_keys":["range%1:14:00::"],"candidates":[{"index":1,"sense_key":"range%1:07:00::","synset_id":"scope.n.01"},{"index":2,"sense_key":"range%1:15:01::","synset_id":"range.n.02"},{"index":3,"sense_key":"range%1:15:02::","synset_id":"range.n.03"},{"index":4,"sense_key":"range%1:17:00::","synset_id":"range.n.04"},{"index":5,"sense_key":"range%1:06:00::","synset_id":"range.n.05"},{"index":6,"sense_key":"range%1:14:00::","synset_id":"range.n.06"},{"index":7,"sense_key":"range%1:14:01::","synset_id":"image.n.07"},{"index":8,"sense_key":"range%1:09:00::","synset_id":"compass.n.03"},{"index":9,"sense_key":"range%1:06:01::","synset_id":"stove.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"range%1:14:00::","call_ids":["semeval2013.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"range%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":872,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.000178,"input_uncached_usd":0.000043599999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.506586375006009}
{"item_id":"semeval2013.d000.s005.t001","gold_sense_keys":["option%1:09:00::"],"candidates":[{"index":1,"sense_key":"option%1:21:00::","synset_id":"option.n.01"},{"index":2,"sense_key":"option%1:09:00::","synset_id":"option.n.02"},{"index":3,"sense_key":"option%1:04:00::","synset_id":"choice.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"option%1:09:00::","call_ids":["semeval2013.d000.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"option%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023774999999999998,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.565173832990695}
{"item_id":"semeval2013.d000.s005.t002","gold_sense_keys":["question%1:10:00::","question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005268,"input_uncached_usd":0.000034,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.926964166981634}
{"item_id":"semeval2013.d000.s005.t003","gold_sense_keys":["upper_limit%1:23:00::"],"candidates":[{"index":1,"sense_key":"upper_limit%1:23:00::","synset_id":"maximum.n.01"},{"index":2,"sense_key":"upper_limit%1:15:00::","synset_id":"upper_limit.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"upper_limit%1:23:00::","call_ids":["semeval2013.d000.s005.t003__v1__a1","semeval2013.d000.s005.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"upper_limit%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005205499999999999,"input_uncached_usd":0.00004695,"input_cached_usd":0.0,"output_usd":0.00047359999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.70287279196782}
{"item_id":"semeval2013.d000.s005.t004","gold_sense_keys":["temperature%1:07:00::"],"candidates":[{"index":1,"sense_key":"temperature%1:07:00::","synset_id":"temperature.n.01"},{"index":2,"sense_key":"temperature%1:09:00::","synset_id":"temperature.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"temperature%1:07:00::","call_ids":["semeval2013.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"temperature%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004129,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.395971000019927}
{"item_id":"semeval2013.d000.s005.t005","gold_sense_keys":["rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":2,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":3,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":4,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":5,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":6,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":9,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":10,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"rise%1:11:02::","call_ids":["semeval2013.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"rise%1:11:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":796,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048139999999999994,"input_uncached_usd":0.0000398,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.38516595895635}
{"item_id":"semeval2013.d000.s006.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005563,"input_uncached_usd":0.0000379,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.879401082987897}
{"item_id":"semeval2013.d000.s006.t001","gold_sense_keys":["flexibility%1:07:01::"],"candidates":[{"index":1,"sense_key":"flexibility%1:07:02::","synset_id":"flexibility.n.01"},{"index":2,"sense_key":"flexibility%1:07:01::","synset_id":"flexibility.n.02"},{"index":3,"sense_key":"flexibility%1:07:03::","synset_id":"tractability.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"flexibility%1:07:01::","call_ids":["semeval2013.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"flexibility%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023789999999999998,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.66354629205307}
{"item_id":"semeval2013.d000.s006.t002","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":2,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":3,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":4,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":5,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":6,"sense_key":"process%1:03:00::","synset_id":"process.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"process%1:04:00::","call_ids":["semeval2013.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"process%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045224999999999996,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.649084084026981}
{"item_id":"semeval2013.d000.s006.t003","gold_sense_keys":["washington%1:15:01::"],"candidates":[{"index":1,"sense_key":"washington%1:15:01::","synset_id":"washington.n.01"},{"index":2,"sense_key":"washington%1:15:00::","synset_id":"washington.n.02"},{"index":3,"sense_key":"washington%1:14:00::","synset_id":"capital.n.06"},{"index":4,"sense_key":"washington%1:18:00::","synset_id":"washington.n.04"},{"index":5,"sense_key":"washington%1:18:01::","synset_id":"washington.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"washington%1:15:00::","call_ids":["semeval2013.d000.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"washington%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002962,"input_uncached_usd":0.0000334,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.703416916017886}
{"item_id":"semeval2013.d000.s006.t004","gold_sense_keys":["representative%1:18:00::","representative%1:18:02::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"representative%1:18:02::","call_ids":["semeval2013.d000.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"representative%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004969,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.874089666991495}
{"item_id":"semeval2013.d000.s007.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2013.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021279999999999997,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7690769999753684}
{"item_id":"semeval2013.d000.s007.t001","gold_sense_keys":["negotiation%1:04:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"negotiation%1:04:00::","call_ids":["semeval2013.d000.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"negotiation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026189999999999997,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.165703667036723}
{"item_id":"semeval2013.d000.s007.t002","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"text%1:10:01::","call_ids":["semeval2013.d000.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"text%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034435,"input_uncached_usd":0.000030749999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.82377662503859}
{"item_id":"semeval2013.d000.s007.t003","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s007.t003__v1__a1","semeval2013.d000.s007.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1083,"cached_input_tokens":0,"output_tokens":3296,"reasoning_output_tokens":3264},"cost":{"currency":"USD","total_usd":0.00137255,"input_uncached_usd":0.00005415,"input_cached_usd":0.0,"output_usd":0.0013184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":22.85370225092629}
{"item_id":"semeval2013.d000.s007.t004","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021009999999999998,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.035300624964293}
{"item_id":"semeval2013.d000.s007.t005","gold_sense_keys":["picture%1:26:00::"],"candidates":[{"index":1,"sense_key":"picture%1:06:00::","synset_id":"picture.n.01"},{"index":2,"sense_key":"picture%1:06:01::","synset_id":"painting.n.01"},{"index":3,"sense_key":"picture%1:09:00::","synset_id":"mental_picture.n.01"},{"index":4,"sense_key":"picture%1:26:00::","synset_id":"picture.n.04"},{"index":5,"sense_key":"picture%1:10:00::","synset_id":"picture.n.05"},{"index":6,"sense_key":"picture%1:10:02::","synset_id":"movie.n.01"},{"index":7,"sense_key":"picture%1:10:01::","synset_id":"video.n.01"},{"index":8,"sense_key":"picture%1:10:03::","synset_id":"word_picture.n.01"},{"index":9,"sense_key":"picture%1:09:01::","synset_id":"picture.n.09"},{"index":10,"sense_key":"picture%1:06:02::","synset_id":"photograph.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"picture%1:26:00::","call_ids":["semeval2013.d000.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"picture%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00054045,"input_uncached_usd":0.00004765,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.630410166981164}
{"item_id":"semeval2013.d000.s008.t000","gold_sense_keys":["outline%1:09:00::"],"candidates":[{"index":1,"sense_key":"outline%1:15:00::","synset_id":"outline.n.01"},{"index":2,"sense_key":"outline%1:10:00::","synset_id":"outline.n.02"},{"index":3,"sense_key":"outline%1:09:00::","synset_id":"outline.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"outline%1:09:00::","call_ids":["semeval2013.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"outline%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021065,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.603937708947342}
{"item_id":"semeval2013.d000.s008.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":799,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027675,"input_uncached_usd":0.000039949999999999995,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.363560916972347}
{"item_id":"semeval2013.d000.s008.t002","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029085,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.824508749996312}
{"item_id":"semeval2013.d000.s008.t004","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d000.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":767,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035194999999999996,"input_uncached_usd":0.00003835,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.216535458981525}
{"item_id":"semeval2013.d000.s008.t006","gold_sense_keys":["period%1:28:00::"],"candidates":[{"index":1,"sense_key":"period%1:28:00::","synset_id":"time_period.n.01"},{"index":2,"sense_key":"period%1:28:02::","synset_id":"period.n.02"},{"index":3,"sense_key":"period%1:28:07::","synset_id":"period.n.03"},{"index":4,"sense_key":"period%1:28:03::","synset_id":"period.n.04"},{"index":5,"sense_key":"period%1:28:04::","synset_id":"period.n.05"},{"index":6,"sense_key":"period%1:22:00::","synset_id":"menstruation.n.01"},{"index":7,"sense_key":"period%1:10:00::","synset_id":"period.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"period%1:28:00::","call_ids":["semeval2013.d000.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"period%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040264999999999995,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.96738283400191}
{"item_id":"semeval2013.d000.s009.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037299999999999996,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.548191040987149}
{"item_id":"semeval2013.d000.s009.t001","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003175,"input_uncached_usd":0.0000295,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.007534583972301}
{"item_id":"semeval2013.d000.s010.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024139999999999999,"input_uncached_usd":0.0000302,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.069870584004093}
{"item_id":"semeval2013.d000.s010.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034379999999999995,"input_uncached_usd":0.0000302,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.148368666006718}
{"item_id":"semeval2013.d000.s010.t006","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047484999999999997,"input_uncached_usd":0.00003285,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.666950458020438}
{"item_id":"semeval2013.d000.s010.t007","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003164,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.839131916000042}
{"item_id":"semeval2013.d000.s010.t009","gold_sense_keys":["flood%1:19:00::"],"candidates":[{"index":1,"sense_key":"flood%1:19:00::","synset_id":"flood.n.01"},{"index":2,"sense_key":"flood%1:23:00::","synset_id":"flood.n.02"},{"index":3,"sense_key":"flood%1:06:00::","synset_id":"flood.n.03"},{"index":4,"sense_key":"flood%1:04:01::","synset_id":"flood.n.04"},{"index":5,"sense_key":"flood%1:04:00::","synset_id":"flood.n.05"},{"index":6,"sense_key":"flood%1:11:00::","synset_id":"flood_tide.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"flood%1:19:00::","call_ids":["semeval2013.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"flood%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029644999999999997,"input_uncached_usd":0.00003365,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.544730083027389}
{"item_id":"semeval2013.d000.s010.t010","gold_sense_keys":["drought%1:26:00::"],"candidates":[{"index":1,"sense_key":"drought%1:26:00::","synset_id":"drought.n.01"},{"index":2,"sense_key":"drought%1:28:00::","synset_id":"drought.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"drought%1:26:00::","call_ids":["semeval2013.d000.s010.t010__v1__a1","semeval2013.d000.s010.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"drought%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1033,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00052525,"input_uncached_usd":0.000051649999999999995,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.225188167067245}
{"item_id":"semeval2013.d000.s011.t001","gold_sense_keys":["sort%1:09:00::"],"candidates":[{"index":1,"sense_key":"sort%1:09:00::","synset_id":"kind.n.01"},{"index":2,"sense_key":"sort%1:07:00::","synset_id":"sort.n.02"},{"index":3,"sense_key":"sort%1:18:00::","synset_id":"sort.n.03"},{"index":4,"sense_key":"sort%1:22:00::","synset_id":"sort.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sort%1:09:00::","call_ids":["semeval2013.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sort%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026605,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.71607933298219}
{"item_id":"semeval2013.d000.s011.t003","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006208,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.00652216700837}
{"item_id":"semeval2013.d000.s012.t001","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"climate%1:26:01::","call_ids":["semeval2013.d000.s012.t001__v1__a1","semeval2013.d000.s012.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"climate%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1057,"cached_input_tokens":0,"output_tokens":3809,"reasoning_output_tokens":3776},"cost":{"currency":"USD","total_usd":0.0015764499999999999,"input_uncached_usd":0.00005285,"input_cached_usd":0.0,"output_usd":0.0015236,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":22.903133083949797}
{"item_id":"semeval2013.d000.s012.t002","gold_sense_keys":["envoy%1:18:00::","envoy%1:18:01::"],"candidates":[{"index":1,"sense_key":"envoy%1:18:00::","synset_id":"envoy.n.01"},{"index":2,"sense_key":"envoy%1:18:01::","synset_id":"emissary.n.01"},{"index":3,"sense_key":"envoy%1:10:00::","synset_id":"envoy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"envoy%1:18:00::","call_ids":["semeval2013.d000.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"envoy%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005438,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.121307499997783}
{"item_id":"semeval2013.d000.s012.t004","gold_sense_keys":["cut%1:04:05::"],"candidates":[{"index":1,"sense_key":"cut%1:21:00::","synset_id":"cut.n.01"},{"index":2,"sense_key":"cut%1:10:00::","synset_id":"cut.n.02"},{"index":3,"sense_key":"cut%1:06:00::","synset_id":"cut.n.03"},{"index":4,"sense_key":"cut%1:26:01::","synset_id":"cut.n.04"},{"index":5,"sense_key":"cut%1:26:00::","synset_id":"cut.n.05"},{"index":6,"sense_key":"cut%1:13:00::","synset_id":"cut.n.06"},{"index":7,"sense_key":"cut%1:10:02::","synset_id":"stinger.n.02"},{"index":8,"sense_key":"cut%1:10:04::","synset_id":"cut.n.08"},{"index":9,"sense_key":"cut%1:10:01::","synset_id":"deletion.n.03"},{"index":10,"sense_key":"cut%1:09:00::","synset_id":"cut.n.10"},{"index":11,"sense_key":"cut%1:06:01::","synset_id":"cut.n.11"},{"index":12,"sense_key":"cut%1:04:03::","synset_id":"snub.n.02"},{"index":13,"sense_key":"cut%1:04:07::","synset_id":"baseball_swing.n.01"},{"index":14,"sense_key":"cut%1:04:08::","synset_id":"cut.n.14"},{"index":15,"sense_key":"cut%1:04:09::","synset_id":"cut.n.15"},{"index":16,"sense_key":"cut%1:04:02::","synset_id":"cut.n.16"},{"index":17,"sense_key":"cut%1:04:01::","synset_id":"cut.n.17"},{"index":18,"sense_key":"cut%1:04:00::","synset_id":"cut.n.18"},{"index":19,"sense_key":"cut%1:04:05::","synset_id":"cut.n.19"},{"index":20,"sense_key":"cut%1:04:06::","synset_id":"cut.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"cut%1:04:05::","call_ids":["semeval2013.d000.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"cut%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1200,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002712,"input_uncached_usd":0.000059999999999999995,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.096435624989681}
{"item_id":"semeval2013.d000.s012.t005","gold_sense_keys":["emission%1:04:00::","emission%1:27:00::"],"candidates":[{"index":1,"sense_key":"emission%1:04:00::","synset_id":"emission.n.01"},{"index":2,"sense_key":"emission%1:27:00::","synset_id":"discharge.n.03"},{"index":3,"sense_key":"emission%1:22:00::","synset_id":"emission.n.03"},{"index":4,"sense_key":"emission%1:22:01::","synset_id":"discharge.n.04"},{"index":5,"sense_key":"emission%1:11:00::","synset_id":"emission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"emission%1:27:00::","call_ids":["semeval2013.d000.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"emission%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001628,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1566096249734983}
{"item_id":"semeval2013.d000.s012.t006","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d000.s012.t006__v1__a1","semeval2013.d000.s012.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1055,"cached_input_tokens":0,"output_tokens":4129,"reasoning_output_tokens":4096},"cost":{"currency":"USD","total_usd":0.0017043499999999999,"input_uncached_usd":0.000052749999999999995,"input_cached_usd":0.0,"output_usd":0.0016516,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":27.524722417001612}
{"item_id":"semeval2013.d000.s013.t000","gold_sense_keys":["move%1:04:00::"],"candidates":[{"index":1,"sense_key":"move%1:04:00::","synset_id":"move.n.01"},{"index":2,"sense_key":"move%1:04:01::","synset_id":"move.n.02"},{"index":3,"sense_key":"move%1:04:03::","synset_id":"motion.n.03"},{"index":4,"sense_key":"move%1:04:04::","synset_id":"motion.n.06"},{"index":5,"sense_key":"move%1:04:02::","synset_id":"move.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"move%1:04:00::","call_ids":["semeval2013.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"move%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049895,"input_uncached_usd":0.00003175,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.65308645798359}
{"item_id":"semeval2013.d000.s013.t002","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021495,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.834335125051439}
{"item_id":"semeval2013.d000.s013.t003","gold_sense_keys":["bush_administration%1:14:01::"],"candidates":[{"index":1,"sense_key":"bush_administration%1:14:00::","synset_id":"bush_administration.n.01"},{"index":2,"sense_key":"bush_administration%1:14:01::","synset_id":"bush_administration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bush_administration%1:14:01::","call_ids":["semeval2013.d000.s013.t003__v1__a1","semeval2013.d000.s013.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bush_administration%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":2016,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00085515,"input_uncached_usd":0.00004874999999999999,"input_cached_usd":0.0,"output_usd":0.0008063999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.897131750010885}
{"item_id":"semeval2013.d000.s013.t004","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044035,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.248340749996714}
{"item_id":"semeval2013.d000.s013.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d000.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024124999999999998,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5244146669865586}
{"item_id":"semeval2013.d000.s014.t000","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:14:00::","call_ids":["semeval2013.d000.s014.t000__v1__a1","semeval2013.d000.s014.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":1824,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007848499999999999,"input_uncached_usd":0.00005525,"input_cached_usd":0.0,"output_usd":0.0007296,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.119413750013337}
{"item_id":"semeval2013.d000.s014.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":800,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002768,"input_uncached_usd":0.000039999999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.351968957984354}
{"item_id":"semeval2013.d000.s014.t004","gold_sense_keys":["text%1:10:00::","text%1:10:01::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2013.d000.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052295,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.572316750010941}
{"item_id":"semeval2013.d000.s014.t005","gold_sense_keys":["cut%1:04:05::"],"candidates":[{"index":1,"sense_key":"cut%1:21:00::","synset_id":"cut.n.01"},{"index":2,"sense_key":"cut%1:10:00::","synset_id":"cut.n.02"},{"index":3,"sense_key":"cut%1:06:00::","synset_id":"cut.n.03"},{"index":4,"sense_key":"cut%1:26:01::","synset_id":"cut.n.04"},{"index":5,"sense_key":"cut%1:26:00::","synset_id":"cut.n.05"},{"index":6,"sense_key":"cut%1:13:00::","synset_id":"cut.n.06"},{"index":7,"sense_key":"cut%1:10:02::","synset_id":"stinger.n.02"},{"index":8,"sense_key":"cut%1:10:04::","synset_id":"cut.n.08"},{"index":9,"sense_key":"cut%1:10:01::","synset_id":"deletion.n.03"},{"index":10,"sense_key":"cut%1:09:00::","synset_id":"cut.n.10"},{"index":11,"sense_key":"cut%1:06:01::","synset_id":"cut.n.11"},{"index":12,"sense_key":"cut%1:04:03::","synset_id":"snub.n.02"},{"index":13,"sense_key":"cut%1:04:07::","synset_id":"baseball_swing.n.01"},{"index":14,"sense_key":"cut%1:04:08::","synset_id":"cut.n.14"},{"index":15,"sense_key":"cut%1:04:09::","synset_id":"cut.n.15"},{"index":16,"sense_key":"cut%1:04:02::","synset_id":"cut.n.16"},{"index":17,"sense_key":"cut%1:04:01::","synset_id":"cut.n.17"},{"index":18,"sense_key":"cut%1:04:00::","synset_id":"cut.n.18"},{"index":19,"sense_key":"cut%1:04:05::","synset_id":"cut.n.19"},{"index":20,"sense_key":"cut%1:04:06::","synset_id":"cut.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"cut%1:04:05::","call_ids":["semeval2013.d000.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"cut%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1201,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00050165,"input_uncached_usd":0.000060049999999999996,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.417422666971106}
{"item_id":"semeval2013.d000.s014.t006","gold_sense_keys":["observer%1:18:01::"],"candidates":[{"index":1,"sense_key":"observer%1:18:00::","synset_id":"perceiver.n.01"},{"index":2,"sense_key":"observer%1:18:01::","synset_id":"observer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"observer%1:18:00::","call_ids":["semeval2013.d000.s014.t006__v1__a1","semeval2013.d000.s014.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"observer%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1001,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00067725,"input_uncached_usd":0.00005005,"input_cached_usd":0.0,"output_usd":0.0006272,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.892616083030589}
{"item_id":"semeval2013.d000.s015.t000","gold_sense_keys":["comment%1:10:00::"],"candidates":[{"index":1,"sense_key":"comment%1:10:00::","synset_id":"remark.n.01"},{"index":2,"sense_key":"comment%1:10:01::","synset_id":"comment.n.02"},{"index":3,"sense_key":"comment%1:10:02::","synset_id":"gossip.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"comment%1:10:00::","call_ids":["semeval2013.d000.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"comment%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":561,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039325,"input_uncached_usd":0.000028049999999999997,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.901164791954216}
{"item_id":"semeval2013.d000.s015.t001","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031675000000000003,"input_uncached_usd":0.000028749999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.132062083983328}
{"item_id":"semeval2013.d000.s015.t003","gold_sense_keys":["america%1:15:00::"],"candidates":[{"index":1,"sense_key":"america%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"america%1:17:00::","synset_id":"america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"america%1:15:00::","call_ids":["semeval2013.d000.s015.t003__v1__a1","semeval2013.d000.s015.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"america%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1079,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00070675,"input_uncached_usd":0.00005395,"input_cached_usd":0.0,"output_usd":0.0006528,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.654692166950554}
{"item_id":"semeval2013.d000.s015.t004","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"climate%1:26:00::","call_ids":["semeval2013.d000.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"climate%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008515,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.034167333971709}
{"item_id":"semeval2013.d000.s015.t007","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016074999999999998,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.076462375000119}
{"item_id":"semeval2013.d000.s015.t008","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:14:00::","call_ids":["semeval2013.d000.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039184999999999996,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.899330292013474}
{"item_id":"semeval2013.d000.s015.t009","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003432,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.942302707990166}
{"item_id":"semeval2013.d000.s016.t001","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"world%1:17:00::","call_ids":["semeval2013.d000.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"world%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":3792,"reasoning_output_tokens":3776},"cost":{"currency":"USD","total_usd":0.0015573999999999998,"input_uncached_usd":0.0000406,"input_cached_usd":0.0,"output_usd":0.0015168,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":21.106621834042016}
{"item_id":"semeval2013.d000.s017.t000","gold_sense_keys":["battle%1:04:02::"],"candidates":[{"index":1,"sense_key":"battle%1:04:00::","synset_id":"battle.n.01"},{"index":2,"sense_key":"battle%1:04:01::","synset_id":"struggle.n.01"},{"index":3,"sense_key":"battle%1:04:02::","synset_id":"conflict.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"battle%1:04:01::","call_ids":["semeval2013.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"battle%1:04:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029374999999999996,"input_uncached_usd":0.000031349999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.736737416009419}
{"item_id":"semeval2013.d000.s017.t001","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d000.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026754999999999997,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.264695666031912}
{"item_id":"semeval2013.d000.s017.t003","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039585,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.446090665995143}
{"item_id":"semeval2013.d000.s017.t004","gold_sense_keys":["united_states%1:14:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"united_states%1:14:00::","call_ids":["semeval2013.d000.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"united_states%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007773,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.780914083006792}
{"item_id":"semeval2013.d000.s017.t005","gold_sense_keys":["obligation%1:04:00::"],"candidates":[{"index":1,"sense_key":"obligation%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"obligation%1:26:00::","synset_id":"obligation.n.02"},{"index":3,"sense_key":"obligation%1:24:00::","synset_id":"obligation.n.03"},{"index":4,"sense_key":"obligation%1:21:00::","synset_id":"debt_instrument.n.01"},{"index":5,"sense_key":"obligation%1:10:00::","synset_id":"obligation.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"obligation%1:26:00::","call_ids":["semeval2013.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"obligation%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00060325,"input_uncached_usd":0.00003365,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.084407249989454}
{"item_id":"semeval2013.d000.s017.t006","gold_sense_keys":["nation%1:14:00::"],"candidates":[{"index":1,"sense_key":"nation%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"nation%1:14:01::","synset_id":"nation.n.02"},{"index":3,"sense_key":"nation%1:18:00::","synset_id":"nation.n.03"},{"index":4,"sense_key":"nation%1:14:02::","synset_id":"nation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nation%1:14:00::","call_ids":["semeval2013.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031995,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.951468833955005}
{"item_id":"semeval2013.d000.s017.t007","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"world%1:17:02::","call_ids":["semeval2013.d000.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"world%1:17:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006378,"input_uncached_usd":0.000042199999999999996,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.78444237500662}
{"item_id":"semeval2013.d000.s017.t009","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":696,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029719999999999996,"input_uncached_usd":0.0000348,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4962735419976525}
{"item_id":"semeval2013.d000.s017.t010","gold_sense_keys":["imperative%1:04:00::"],"candidates":[{"index":1,"sense_key":"imperative%1:24:00::","synset_id":"imperative_mood.n.01"},{"index":2,"sense_key":"imperative%1:04:00::","synset_id":"imperative.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"imperative%1:04:00::","call_ids":["semeval2013.d000.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"imperative%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.000493,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.301608750014566}
{"item_id":"semeval2013.d000.s018.t002","gold_sense_keys":["cost%1:21:00::"],"candidates":[{"index":1,"sense_key":"cost%1:21:00::","synset_id":"cost.n.01"},{"index":2,"sense_key":"cost%1:07:00::","synset_id":"monetary_value.n.01"},{"index":3,"sense_key":"cost%1:07:01::","synset_id":"price.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"cost%1:07:01::","call_ids":["semeval2013.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"cost%1:07:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006762,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.187230208946858}
{"item_id":"semeval2013.d000.s018.t004","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00065545,"input_uncached_usd":0.000034649999999999995,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.78105366701493}
{"item_id":"semeval2013.d000.s018.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d000.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003074,"input_uncached_usd":0.000044999999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.101251125044655}
{"item_id":"semeval2013.d000.s019.t003","gold_sense_keys":["central_america%1:14:00::"],"candidates":[{"index":1,"sense_key":"central_america%1:15:00::","synset_id":"central_america.n.01"},{"index":2,"sense_key":"central_america%1:14:00::","synset_id":"central_america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"central_america%1:14:00::","call_ids":["semeval2013.d000.s019.t003__v1__a1","semeval2013.d000.s019.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"central_america%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1113,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00073405,"input_uncached_usd":0.00005565,"input_cached_usd":0.0,"output_usd":0.0006784,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.654358000960201}
{"item_id":"semeval2013.d000.s020.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029695,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8063421249971725}
{"item_id":"semeval2013.d000.s020.t001","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005229000000000001,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.95987762499135}
{"item_id":"semeval2013.d000.s020.t005","gold_sense_keys":["participant%1:18:00::"],"candidates":[{"index":1,"sense_key":"participant%1:18:00::","synset_id":"participant.n.01"},{"index":2,"sense_key":"participant%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participant%1:18:00::","call_ids":["semeval2013.d000.s020.t005__v1__a1","semeval2013.d000.s020.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1041,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00057685,"input_uncached_usd":0.00005204999999999999,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.87123683304526}
{"item_id":"semeval2013.d000.s020.t007","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030419999999999997,"input_uncached_usd":0.0000418,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.600664500030689}
{"item_id":"semeval2013.d000.s021.t000","gold_sense_keys":["member%1:14:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"member%1:14:00::","call_ids":["semeval2013.d000.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"member%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032260000000000004,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.507381375005934}
{"item_id":"semeval2013.d000.s021.t001","gold_sense_keys":["draft%1:10:00::"],"candidates":[{"index":1,"sense_key":"draft%1:21:00::","synset_id":"draft.n.01"},{"index":2,"sense_key":"draft%1:19:00::","synset_id":"draft.n.02"},{"index":3,"sense_key":"draft%1:06:01::","synset_id":"draft.n.03"},{"index":4,"sense_key":"draft%1:13:00::","synset_id":"draft.n.04"},{"index":5,"sense_key":"draft%1:10:00::","synset_id":"draft.n.05"},{"index":6,"sense_key":"draft%1:07:00::","synset_id":"draft.n.06"},{"index":7,"sense_key":"draft%1:06:02::","synset_id":"draft.n.07"},{"index":8,"sense_key":"draft%1:06:00::","synset_id":"draft.n.08"},{"index":9,"sense_key":"draft%1:04:03::","synset_id":"conscription.n.01"},{"index":10,"sense_key":"draft%1:04:02::","synset_id":"gulp.n.01"},{"index":11,"sense_key":"draft%1:04:00::","synset_id":"draft.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"draft%1:10:00::","call_ids":["semeval2013.d000.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"draft%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003815499999999999,"input_uncached_usd":0.00004235,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.469193166994955}
{"item_id":"semeval2013.d000.s021.t003","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2013.d000.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026369999999999996,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.587072875001468}
{"item_id":"semeval2013.d000.s021.t005","gold_sense_keys":["morning%1:28:00::"],"candidates":[{"index":1,"sense_key":"morning%1:28:00::","synset_id":"morning.n.01"},{"index":2,"sense_key":"morning%1:10:00::","synset_id":"good_morning.n.01"},{"index":3,"sense_key":"morning%1:28:01::","synset_id":"dawn.n.01"},{"index":4,"sense_key":"morning%1:11:00::","synset_id":"dawn.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"morning%1:28:00::","call_ids":["semeval2013.d000.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"morning%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001649,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.798555374960415}
{"item_id":"semeval2013.d000.s022.t000","gold_sense_keys":["climate%1:26:00::"],"candidates":[{"index":1,"sense_key":"climate%1:26:00::","synset_id":"climate.n.01"},{"index":2,"sense_key":"climate%1:26:01::","synset_id":"climate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"climate%1:26:01::","call_ids":["semeval2013.d000.s022.t000__v1__a1","semeval2013.d000.s022.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"climate%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1133,"cached_input_tokens":0,"output_tokens":4384,"reasoning_output_tokens":4352},"cost":{"currency":"USD","total_usd":0.00181025,"input_uncached_usd":0.000056649999999999995,"input_cached_usd":0.0,"output_usd":0.0017536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":26.51353791600559}
{"item_id":"semeval2013.d000.s022.t001","gold_sense_keys":["negotiation%1:10:00::"],"candidates":[{"index":1,"sense_key":"negotiation%1:10:00::","synset_id":"negotiation.n.01"},{"index":2,"sense_key":"negotiation%1:04:00::","synset_id":"negotiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"negotiation%1:04:00::","call_ids":["semeval2013.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"negotiation%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004935499999999999,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.54603650001809}
{"item_id":"semeval2013.d000.s022.t002","gold_sense_keys":["behalf%1:04:00::"],"candidates":[{"index":1,"sense_key":"behalf%1:04:00::","synset_id":"behalf.n.01"},{"index":2,"sense_key":"behalf%1:07:00::","synset_id":"behalf.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behalf%1:04:00::","call_ids":["semeval2013.d000.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behalf%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023825,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.568540000007488}
{"item_id":"semeval2013.d000.s022.t004","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003515,"input_uncached_usd":0.0000379,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.70802195899887}
{"item_id":"semeval2013.d000.s022.t005","gold_sense_keys":["process%1:04:00::"],"candidates":[{"index":1,"sense_key":"process%1:04:00::","synset_id":"procedure.n.01"},{"index":2,"sense_key":"process%1:09:00::","synset_id":"process.n.02"},{"index":3,"sense_key":"process%1:10:00::","synset_id":"summons.n.03"},{"index":4,"sense_key":"process%1:09:01::","synset_id":"process.n.04"},{"index":5,"sense_key":"process%1:08:00::","synset_id":"process.n.05"},{"index":6,"sense_key":"process%1:03:00::","synset_id":"process.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"process%1:04:00::","call_ids":["semeval2013.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"process%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032525,"input_uncached_usd":0.00003725,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.23666258295998}
{"item_id":"semeval2013.d000.s022.t006","gold_sense_keys":["action%1:04:04::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006914,"input_uncached_usd":0.000044999999999999996,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.898798665963113}
{"item_id":"semeval2013.d000.s022.t007","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d000.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019084999999999998,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7258261250099167}
{"item_id":"semeval2013.d000.s023.t001","gold_sense_keys":["urgency%1:07:00::","urgency%1:26:00::"],"candidates":[{"index":1,"sense_key":"urgency%1:26:00::","synset_id":"urgency.n.01"},{"index":2,"sense_key":"urgency%1:07:00::","synset_id":"urgency.n.02"},{"index":3,"sense_key":"urgency%1:26:01::","synset_id":"urgency.n.03"},{"index":4,"sense_key":"urgency%1:10:00::","synset_id":"importunity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"urgency%1:26:01::","call_ids":["semeval2013.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"urgency%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006514499999999999,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.49083783297101}
{"item_id":"semeval2013.d000.s023.t003","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00067745,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.832053042016923}
{"item_id":"semeval2013.d000.s023.t004","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":990,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003375,"input_uncached_usd":0.0000495,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.882798290986102}
{"item_id":"semeval2013.d000.s023.t005","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d000.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":840,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020199999999999998,"input_uncached_usd":0.000042,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.924680874974001}
{"item_id":"semeval2013.d000.s024.t000","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2013.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049405,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.754990166984499}
{"item_id":"semeval2013.d000.s024.t001","gold_sense_keys":["minister%1:18:02::"],"candidates":[{"index":1,"sense_key":"minister%1:18:00::","synset_id":"curate.n.01"},{"index":2,"sense_key":"minister%1:18:02::","synset_id":"minister.n.02"},{"index":3,"sense_key":"minister%1:18:01::","synset_id":"minister.n.03"},{"index":4,"sense_key":"minister%1:04:00::","synset_id":"minister.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minister%1:18:02::","call_ids":["semeval2013.d000.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minister%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003182,"input_uncached_usd":0.0000302,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.336195667041466}
{"item_id":"semeval2013.d000.s024.t002","gold_sense_keys":["plane%1:06:01::"],"candidates":[{"index":1,"sense_key":"plane%1:06:01::","synset_id":"airplane.n.01"},{"index":2,"sense_key":"plane%1:25:00::","synset_id":"plane.n.02"},{"index":3,"sense_key":"plane%1:26:00::","synset_id":"plane.n.03"},{"index":4,"sense_key":"plane%1:06:02::","synset_id":"plane.n.04"},{"index":5,"sense_key":"plane%1:06:00::","synset_id":"plane.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"plane%1:06:01::","call_ids":["semeval2013.d000.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"plane%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029685,"input_uncached_usd":0.00003445,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.599922833964229}
{"item_id":"semeval2013.d000.s024.t003","gold_sense_keys":["hall%1:06:02::","hall%1:06:03::","hall%1:06:05::"],"candidates":[{"index":1,"sense_key":"hall%1:06:03::","synset_id":"hallway.n.01"},{"index":2,"sense_key":"hall%1:06:05::","synset_id":"anteroom.n.01"},{"index":3,"sense_key":"hall%1:06:02::","synset_id":"hall.n.03"},{"index":4,"sense_key":"hall%1:06:04::","synset_id":"dormitory.n.01"},{"index":5,"sense_key":"hall%1:06:06::","synset_id":"manor_hall.n.01"},{"index":6,"sense_key":"hall%1:18:04::","synset_id":"hall.n.06"},{"index":7,"sense_key":"hall%1:18:03::","synset_id":"hall.n.07"},{"index":8,"sense_key":"hall%1:18:02::","synset_id":"hall.n.08"},{"index":9,"sense_key":"hall%1:18:01::","synset_id":"hall.n.09"},{"index":10,"sense_key":"hall%1:18:00::","synset_id":"hall.n.10"},{"index":11,"sense_key":"hall%1:06:07::","synset_id":"mansion.n.02"},{"index":12,"sense_key":"hall%1:06:08::","synset_id":"hall.n.12"},{"index":13,"sense_key":"hall%1:06:01::","synset_id":"hall.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hall%1:06:03::","call_ids":["semeval2013.d000.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hall%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008230500000000001,"input_uncached_usd":0.00004865,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.646876041020732}
{"item_id":"semeval2013.d000.s024.t004","gold_sense_keys":["meeting%1:14:00::"],"candidates":[{"index":1,"sense_key":"meeting%1:14:00::","synset_id":"meeting.n.01"},{"index":2,"sense_key":"meeting%1:14:01::","synset_id":"meeting.n.02"},{"index":3,"sense_key":"meeting%1:11:00::","synset_id":"meeting.n.03"},{"index":4,"sense_key":"meeting%1:04:00::","synset_id":"meeting.n.04"},{"index":5,"sense_key":"meeting%1:04:02::","synset_id":"merging.n.01"},{"index":6,"sense_key":"meeting%1:15:00::","synset_id":"confluence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"meeting%1:14:00::","call_ids":["semeval2013.d000.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"meeting%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005294000000000001,"input_uncached_usd":0.000036599999999999995,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.083736999949906}
{"item_id":"semeval2013.d000.s024.t006","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"claim%1:04:00::","call_ids":["semeval2013.d000.s024.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"claim%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042500000000000003,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.9356115410337225}
{"item_id":"semeval2013.d000.s024.t007","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d000.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018815,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9539874169859104}
{"item_id":"semeval2013.d000.s025.t000","gold_sense_keys":["gathering%1:14:00::"],"candidates":[{"index":1,"sense_key":"gathering%1:14:00::","synset_id":"gathering.n.01"},{"index":2,"sense_key":"gathering%1:04:00::","synset_id":"assembly.n.06"},{"index":3,"sense_key":"gathering%1:04:01::","synset_id":"gather.n.02"},{"index":4,"sense_key":"gathering%1:06:00::","synset_id":"gather.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"gathering%1:14:00::","call_ids":["semeval2013.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"gathering%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029184999999999997,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4389512919588014}
{"item_id":"semeval2013.d000.s025.t001","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2013.d000.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019135,"input_uncached_usd":0.000031349999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.633328082971275}
{"item_id":"semeval2013.d000.s025.t002","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"day%1:28:00::","call_ids":["semeval2013.d000.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043599999999999997,"input_uncached_usd":0.0000456,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.497214625007473}
{"item_id":"semeval2013.d000.s025.t004","gold_sense_keys":["show%1:10:00::"],"candidates":[{"index":1,"sense_key":"show%1:04:00::","synset_id":"show.n.01"},{"index":2,"sense_key":"show%1:10:03::","synset_id":"display.n.01"},{"index":3,"sense_key":"show%1:10:00::","synset_id":"show.n.03"},{"index":4,"sense_key":"show%1:04:01::","synset_id":"appearance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%1:10:00::","call_ids":["semeval2013.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019055,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.246434708999004}
{"item_id":"semeval2013.d000.s025.t005","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003229,"input_uncached_usd":0.0000349,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.5269300829968415}
{"item_id":"semeval2013.d000.s025.t006","gold_sense_keys":["draft%1:10:00::"],"candidates":[{"index":1,"sense_key":"draft%1:21:00::","synset_id":"draft.n.01"},{"index":2,"sense_key":"draft%1:19:00::","synset_id":"draft.n.02"},{"index":3,"sense_key":"draft%1:06:01::","synset_id":"draft.n.03"},{"index":4,"sense_key":"draft%1:13:00::","synset_id":"draft.n.04"},{"index":5,"sense_key":"draft%1:10:00::","synset_id":"draft.n.05"},{"index":6,"sense_key":"draft%1:07:00::","synset_id":"draft.n.06"},{"index":7,"sense_key":"draft%1:06:02::","synset_id":"draft.n.07"},{"index":8,"sense_key":"draft%1:06:00::","synset_id":"draft.n.08"},{"index":9,"sense_key":"draft%1:04:03::","synset_id":"conscription.n.01"},{"index":10,"sense_key":"draft%1:04:02::","synset_id":"gulp.n.01"},{"index":11,"sense_key":"draft%1:04:00::","synset_id":"draft.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"draft%1:10:00::","call_ids":["semeval2013.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"draft%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":854,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002543,"input_uncached_usd":0.0000427,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.377406165993307}
{"item_id":"semeval2013.d000.s025.t007","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2013.d000.s025.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001361,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.563304374984}
{"item_id":"semeval2013.d000.s025.t008","gold_sense_keys":["paper%1:10:00::"],"candidates":[{"index":1,"sense_key":"paper%1:27:00::","synset_id":"paper.n.01"},{"index":2,"sense_key":"paper%1:10:01::","synset_id":"composition.n.08"},{"index":3,"sense_key":"paper%1:10:03::","synset_id":"newspaper.n.01"},{"index":4,"sense_key":"paper%1:10:00::","synset_id":"paper.n.04"},{"index":5,"sense_key":"paper%1:10:02::","synset_id":"paper.n.05"},{"index":6,"sense_key":"paper%1:14:00::","synset_id":"newspaper.n.02"},{"index":7,"sense_key":"paper%1:06:00::","synset_id":"newspaper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"paper%1:10:00::","call_ids":["semeval2013.d000.s025.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"paper%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.000401,"input_uncached_usd":0.0000362,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2751481250161305}
{"item_id":"semeval2013.d000.s025.t010","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2013.d000.s025.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013895,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3176960829878226}
{"item_id":"semeval2013.d000.s026.t000","gold_sense_keys":["intensity%1:07:00::"],"candidates":[{"index":1,"sense_key":"intensity%1:07:03::","synset_id":"intensity.n.01"},{"index":2,"sense_key":"intensity%1:07:00::","synset_id":"intensity.n.02"},{"index":3,"sense_key":"intensity%1:07:02::","synset_id":"volume.n.06"},{"index":4,"sense_key":"intensity%1:07:01::","synset_id":"saturation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"intensity%1:07:00::","call_ids":["semeval2013.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"intensity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021569999999999998,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.186685834021773}
{"item_id":"semeval2013.d000.s026.t001","gold_sense_keys":["minister%1:18:02::"],"candidates":[{"index":1,"sense_key":"minister%1:18:00::","synset_id":"curate.n.01"},{"index":2,"sense_key":"minister%1:18:02::","synset_id":"minister.n.02"},{"index":3,"sense_key":"minister%1:18:01::","synset_id":"minister.n.03"},{"index":4,"sense_key":"minister%1:04:00::","synset_id":"minister.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"minister%1:18:02::","call_ids":["semeval2013.d000.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"minister%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036805,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.308500582992565}
{"item_id":"semeval2013.d000.s027.t002","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d000.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003405,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.852449666999746}
{"item_id":"semeval2013.d000.s028.t000","gold_sense_keys":["heavyweight%1:18:00::"],"candidates":[{"index":1,"sense_key":"heavyweight%1:18:04::","synset_id":"heavyweight.n.01"},{"index":2,"sense_key":"heavyweight%1:18:02::","synset_id":"heavyweight.n.02"},{"index":3,"sense_key":"heavyweight%1:18:01::","synset_id":"heavyweight.n.03"},{"index":4,"sense_key":"heavyweight%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"heavyweight%1:18:00::","synset_id":"colossus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"heavyweight%1:18:01::","call_ids":["semeval2013.d000.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"heavyweight%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056945,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.33260549994884}
{"item_id":"semeval2013.d001.s000.t001","gold_sense_keys":["israel%1:15:00::"],"candidates":[{"index":1,"sense_key":"israel%1:15:00::","synset_id":"israel.n.01"},{"index":2,"sense_key":"israel%1:15:01::","synset_id":"israel.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"israel%1:15:00::","call_ids":["semeval2013.d001.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"israel%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022754999999999997,"input_uncached_usd":0.000016349999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3944320420268923}
{"item_id":"semeval2013.d001.s001.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s001.t002__v1__a1","semeval2013.d001.s001.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006886500000000001,"input_uncached_usd":0.000035850000000000004,"input_cached_usd":0.0,"output_usd":0.0006528,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.038392832968384}
{"item_id":"semeval2013.d001.s001.t003","gold_sense_keys":["loss%1:04:00::"],"candidates":[{"index":1,"sense_key":"loss%1:21:01::","synset_id":"loss.n.01"},{"index":2,"sense_key":"loss%1:22:00::","synset_id":"loss.n.02"},{"index":3,"sense_key":"loss%1:04:00::","synset_id":"loss.n.03"},{"index":4,"sense_key":"loss%1:07:00::","synset_id":"loss.n.04"},{"index":5,"sense_key":"loss%1:11:01::","synset_id":"loss.n.05"},{"index":6,"sense_key":"loss%1:21:00::","synset_id":"loss.n.06"},{"index":7,"sense_key":"loss%1:11:00::","synset_id":"personnel_casualty.n.01"},{"index":8,"sense_key":"loss%1:11:02::","synset_id":"passing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"loss%1:04:00::","call_ids":["semeval2013.d001.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"loss%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005743,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.850442541996017}
{"item_id":"semeval2013.d001.s001.t004","gold_sense_keys":["performance%1:04:00::","performance%1:04:03::"],"candidates":[{"index":1,"sense_key":"performance%1:10:00::","synset_id":"performance.n.01"},{"index":2,"sense_key":"performance%1:04:01::","synset_id":"performance.n.02"},{"index":3,"sense_key":"performance%1:04:00::","synset_id":"performance.n.03"},{"index":4,"sense_key":"performance%1:04:03::","synset_id":"performance.n.04"},{"index":5,"sense_key":"performance%1:22:00::","synset_id":"operation.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"performance%1:04:03::","call_ids":["semeval2013.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"performance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039125,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.95986087503843}
{"item_id":"semeval2013.d001.s001.t005","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":334,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007143000000000001,"input_uncached_usd":0.0000167,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.601073666010052}
{"item_id":"semeval2013.d001.s001.t006","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"point%1:10:01::","call_ids":["semeval2013.d001.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"point%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1339,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00038055,"input_uncached_usd":0.00006695,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.863237417011987}
{"item_id":"semeval2013.d001.s001.t007","gold_sense_keys":["top%1:26:00::"],"candidates":[{"index":1,"sense_key":"top%1:15:01::","synset_id":"top.n.01"},{"index":2,"sense_key":"top%1:15:00::","synset_id":"top.n.02"},{"index":3,"sense_key":"top%1:15:02::","synset_id":"peak.n.04"},{"index":4,"sense_key":"top%1:28:00::","synset_id":"top.n.04"},{"index":5,"sense_key":"top%1:26:00::","synset_id":"acme.n.01"},{"index":6,"sense_key":"top%1:07:02::","synset_id":"top.n.06"},{"index":7,"sense_key":"top%1:06:04::","synset_id":"top.n.07"},{"index":8,"sense_key":"top%1:06:03::","synset_id":"top.n.08"},{"index":9,"sense_key":"top%1:06:01::","synset_id":"top.n.09"},{"index":10,"sense_key":"top%1:06:00::","synset_id":"top.n.10"},{"index":11,"sense_key":"top%1:06:02::","synset_id":"circus_tent.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"top%1:26:00::","call_ids":["semeval2013.d001.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"top%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":850,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005353000000000001,"input_uncached_usd":0.000042499999999999996,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.29224774998147}
{"item_id":"semeval2013.d001.s001.t008","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d001.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":395,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033335,"input_uncached_usd":0.00001975,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.048735999967903}
{"item_id":"semeval2013.d001.s002.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004852,"input_uncached_usd":0.000018,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.934524165990297}
{"item_id":"semeval2013.d001.s002.t001","gold_sense_keys":["absence%1:04:00::","absence%1:26:00::"],"candidates":[{"index":1,"sense_key":"absence%1:26:00::","synset_id":"absence.n.01"},{"index":2,"sense_key":"absence%1:04:00::","synset_id":"absence.n.02"},{"index":3,"sense_key":"absence%1:28:00::","synset_id":"absence.n.03"},{"index":4,"sense_key":"absence%1:26:01::","synset_id":"absence.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"absence%1:26:00::","call_ids":["semeval2013.d001.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"absence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003875,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.944895667023957}
{"item_id":"semeval2013.d001.s002.t002","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":3,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":4,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involvement%1:04:00::","call_ids":["semeval2013.d001.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026264999999999996,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0937546249479055}
{"item_id":"semeval2013.d001.s002.t004","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":898,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004353,"input_uncached_usd":0.0000449,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.265833791985642}
{"item_id":"semeval2013.d001.s003.t001","gold_sense_keys":["sideline%1:15:00::"],"candidates":[{"index":1,"sense_key":"sideline%1:15:00::","synset_id":"sideline.n.01"},{"index":2,"sense_key":"sideline%1:06:00::","synset_id":"sideline.n.02"},{"index":3,"sense_key":"sideline%1:04:00::","synset_id":"avocation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sideline%1:15:00::","call_ids":["semeval2013.d001.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sideline%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006163000000000001,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.728234832989983}
{"item_id":"semeval2013.d001.s003.t002","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d001.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021864999999999998,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.417340957967099}
{"item_id":"semeval2013.d001.s003.t003","gold_sense_keys":["triumph%1:11:00::"],"candidates":[{"index":1,"sense_key":"triumph%1:11:00::","synset_id":"victory.n.01"},{"index":2,"sense_key":"triumph%1:12:00::","synset_id":"triumph.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"triumph%1:11:00::","call_ids":["semeval2013.d001.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"triumph%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004104,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0047224999871105}
{"item_id":"semeval2013.d001.s004.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d001.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032695,"input_uncached_usd":0.00003895,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.056204874999821}
{"item_id":"semeval2013.d001.s004.t001","gold_sense_keys":["favor%1:07:00::"],"candidates":[{"index":1,"sense_key":"favor%1:04:00::","synset_id":"favor.n.01"},{"index":2,"sense_key":"favor%1:07:00::","synset_id":"favor.n.02"},{"index":3,"sense_key":"favor%1:09:00::","synset_id":"favor.n.03"},{"index":4,"sense_key":"favor%1:12:00::","synset_id":"favor.n.04"},{"index":5,"sense_key":"favor%1:06:00::","synset_id":"party_favor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"favor%1:07:00::","call_ids":["semeval2013.d001.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"favor%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044205,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.082384042034391}
{"item_id":"semeval2013.d001.s005.t000","gold_sense_keys":["basket%1:04:00::"],"candidates":[{"index":1,"sense_key":"basket%1:06:00::","synset_id":"basket.n.01"},{"index":2,"sense_key":"basket%1:23:00::","synset_id":"basket.n.02"},{"index":3,"sense_key":"basket%1:06:01::","synset_id":"basket.n.03"},{"index":4,"sense_key":"basket%1:04:00::","synset_id":"basket.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"basket%1:04:00::","call_ids":["semeval2013.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"basket%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003391,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.359177583013661}
{"item_id":"semeval2013.d001.s005.t001","gold_sense_keys":["perimeter%1:25:00::"],"candidates":[{"index":1,"sense_key":"perimeter%1:25:00::","synset_id":"margin.n.01"},{"index":2,"sense_key":"perimeter%1:25:01::","synset_id":"perimeter.n.02"},{"index":3,"sense_key":"perimeter%1:07:00::","synset_id":"circumference.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perimeter%1:25:00::","call_ids":["semeval2013.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perimeter%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043955,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.484799208003096}
{"item_id":"semeval2013.d001.s005.t002","gold_sense_keys":["speculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"speculation%1:10:03::","synset_id":"guess.n.01"},{"index":2,"sense_key":"speculation%1:09:00::","synset_id":"speculation.n.02"},{"index":3,"sense_key":"speculation%1:21:00::","synset_id":"speculation.n.03"},{"index":4,"sense_key":"speculation%1:09:01::","synset_id":"meditation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"speculation%1:09:00::","call_ids":["semeval2013.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"speculation%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024074999999999997,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9122571669868194}
{"item_id":"semeval2013.d001.s006.t000","gold_sense_keys":["scorer%1:18:00::"],"candidates":[{"index":1,"sense_key":"scorer%1:18:00::","synset_id":"scorer.n.01"},{"index":2,"sense_key":"scorer%1:18:01::","synset_id":"scorer.n.02"},{"index":3,"sense_key":"scorer%1:18:02::","synset_id":"scorekeeper.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scorer%1:18:00::","call_ids":["semeval2013.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scorer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018605,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2251557919662446}
{"item_id":"semeval2013.d001.s006.t001","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1019,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041575,"input_uncached_usd":0.00005095,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.334168041998055}
{"item_id":"semeval2013.d001.s007.t000","gold_sense_keys":["quarter%1:28:01::"],"candidates":[{"index":1,"sense_key":"quarter%1:23:01::","synset_id":"one-fourth.n.01"},{"index":2,"sense_key":"quarter%1:15:00::","synset_id":"quarter.n.02"},{"index":3,"sense_key":"quarter%1:28:01::","synset_id":"quarter.n.03"},{"index":4,"sense_key":"quarter%1:28:03::","synset_id":"quarter.n.04"},{"index":5,"sense_key":"quarter%1:28:00::","synset_id":"quarter.n.05"},{"index":6,"sense_key":"quarter%1:28:02::","synset_id":"quarter.n.06"},{"index":7,"sense_key":"quarter%1:24:00::","synset_id":"quarter.n.07"},{"index":8,"sense_key":"quarter%1:23:02::","synset_id":"quarter.n.08"},{"index":9,"sense_key":"quarter%1:23:04::","synset_id":"quarter.n.09"},{"index":10,"sense_key":"quarter%1:21:00::","synset_id":"quarter.n.10"},{"index":11,"sense_key":"quarter%1:18:00::","synset_id":"quarter.n.11"},{"index":12,"sense_key":"quarter%1:06:00::","synset_id":"stern.n.01"},{"index":13,"sense_key":"quarter%1:06:01::","synset_id":"quarter.n.13"},{"index":14,"sense_key":"quarter%1:04:00::","synset_id":"quarter.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"quarter%1:28:01::","call_ids":["semeval2013.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"quarter%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":906,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00023089999999999997,"input_uncached_usd":0.000045299999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.854242166969925}
{"item_id":"semeval2013.d001.s007.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036204999999999994,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.542362124950159}
{"item_id":"semeval2013.d001.s007.t003","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2013.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037729999999999996,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.569611084007192}
{"item_id":"semeval2013.d001.s007.t004","gold_sense_keys":["whistle%1:10:00::"],"candidates":[{"index":1,"sense_key":"whistle%1:11:00::","synset_id":"whistle.n.01"},{"index":2,"sense_key":"whistle%1:10:00::","synset_id":"whistle.n.02"},{"index":3,"sense_key":"whistle%1:06:02::","synset_id":"whistle.n.03"},{"index":4,"sense_key":"whistle%1:06:01::","synset_id":"whistle.n.04"},{"index":5,"sense_key":"whistle%1:06:00::","synset_id":"pennywhistle.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"whistle%1:10:00::","call_ids":["semeval2013.d001.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"whistle%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004205,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.061008500051685}
{"item_id":"semeval2013.d001.s007.t006","gold_sense_keys":["return%1:04:00::","return%1:04:01::"],"candidates":[{"index":1,"sense_key":"return%1:10:01::","synset_id":"tax_return.n.01"},{"index":2,"sense_key":"return%1:04:01::","synset_id":"return.n.02"},{"index":3,"sense_key":"return%1:11:02::","synset_id":"return.n.03"},{"index":4,"sense_key":"return%1:04:02::","synset_id":"restitution.n.03"},{"index":5,"sense_key":"return%1:04:00::","synset_id":"return.n.05"},{"index":6,"sense_key":"return%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"return%1:11:01::","synset_id":"recurrence.n.01"},{"index":8,"sense_key":"return%1:10:02::","synset_id":"rejoinder.n.01"},{"index":9,"sense_key":"return%1:06:00::","synset_id":"return_key.n.01"},{"index":10,"sense_key":"return%1:04:06::","synset_id":"return.n.10"},{"index":11,"sense_key":"return%1:04:04::","synset_id":"return.n.11"},{"index":12,"sense_key":"return%1:04:05::","synset_id":"return.n.12"},{"index":13,"sense_key":"return%1:04:07::","synset_id":"reappearance.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"return%1:04:01::","call_ids":["semeval2013.d001.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"return%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005927,"input_uncached_usd":0.0000487,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.697605749999639}
{"item_id":"semeval2013.d001.s007.t007","gold_sense_keys":["home%1:15:02::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"home%1:15:04::","call_ids":["semeval2013.d001.s007.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"home%1:15:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00096865,"input_uncached_usd":0.00004065,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.580644791014493}
{"item_id":"semeval2013.d001.s007.t008","gold_sense_keys":["lead%1:23:00::"],"candidates":[{"index":1,"sense_key":"lead%1:07:02::","synset_id":"lead.n.01"},{"index":2,"sense_key":"lead%1:27:00::","synset_id":"lead.n.02"},{"index":3,"sense_key":"lead%1:09:00::","synset_id":"lead.n.03"},{"index":4,"sense_key":"lead%1:04:01::","synset_id":"lead.n.04"},{"index":5,"sense_key":"lead%1:25:00::","synset_id":"lead.n.05"},{"index":6,"sense_key":"lead%1:10:01::","synset_id":"lead.n.06"},{"index":7,"sense_key":"lead%1:23:00::","synset_id":"lead.n.07"},{"index":8,"sense_key":"lead%1:18:00::","synset_id":"star.n.04"},{"index":9,"sense_key":"lead%1:15:00::","synset_id":"lead.n.09"},{"index":10,"sense_key":"lead%1:10:02::","synset_id":"tip.n.03"},{"index":11,"sense_key":"lead%1:10:00::","synset_id":"lead.n.11"},{"index":12,"sense_key":"lead%1:07:01::","synset_id":"spark_advance.n.01"},{"index":13,"sense_key":"lead%1:06:03::","synset_id":"leash.n.01"},{"index":14,"sense_key":"lead%1:06:02::","synset_id":"lead.n.14"},{"index":15,"sense_key":"lead%1:06:00::","synset_id":"lead.n.15"},{"index":16,"sense_key":"lead%1:06:01::","synset_id":"jumper_cable.n.01"},{"index":17,"sense_key":"lead%1:04:00::","synset_id":"lead.n.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"lead%1:23:00::","call_ids":["semeval2013.d001.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"lead%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1110,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00024109999999999998,"input_uncached_usd":0.0000555,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3845619590138085}
{"item_id":"semeval2013.d001.s008.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046479999999999997,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.967868249979801}
{"item_id":"semeval2013.d001.s008.t001","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1468,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002846,"input_uncached_usd":0.0000734,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.935588624968659}
{"item_id":"semeval2013.d001.s008.t002","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d001.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1469,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00023344999999999998,"input_uncached_usd":0.00007345,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9154053750098683}
{"item_id":"semeval2013.d001.s008.t003","gold_sense_keys":["night%1:28:03::"],"candidates":[{"index":1,"sense_key":"night%1:28:00::","synset_id":"night.n.01"},{"index":2,"sense_key":"night%1:28:01::","synset_id":"night.n.02"},{"index":3,"sense_key":"night%1:28:04::","synset_id":"night.n.03"},{"index":4,"sense_key":"night%1:28:02::","synset_id":"night.n.04"},{"index":5,"sense_key":"night%1:26:00::","synset_id":"night.n.05"},{"index":6,"sense_key":"night%1:28:05::","synset_id":"night.n.06"},{"index":7,"sense_key":"night%1:28:03::","synset_id":"night.n.07"},{"index":8,"sense_key":"night%1:18:00::","synset_id":"nox.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"night%1:28:03::","call_ids":["semeval2013.d001.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"night%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047454999999999996,"input_uncached_usd":0.00003295,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.062378957984038}
{"item_id":"semeval2013.d001.s009.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032935,"input_uncached_usd":0.000041349999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.092565250000916}
{"item_id":"semeval2013.d001.s009.t003","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d001.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027275,"input_uncached_usd":0.00003595,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.244673291046638}
{"item_id":"semeval2013.d001.s009.t004","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038054999999999995,"input_uncached_usd":0.000041349999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.584853291045874}
{"item_id":"semeval2013.d001.s010.t000","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["semeval2013.d001.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004192,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.929217208002228}
{"item_id":"semeval2013.d001.s012.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026869999999999997,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.221636582980864}
{"item_id":"semeval2013.d001.s012.t001","gold_sense_keys":["couple%1:23:01::"],"candidates":[{"index":1,"sense_key":"couple%1:14:00::","synset_id":"couple.n.01"},{"index":2,"sense_key":"couple%1:14:01::","synset_id":"couple.n.02"},{"index":3,"sense_key":"couple%1:23:01::","synset_id":"couple.n.03"},{"index":4,"sense_key":"couple%1:23:00::","synset_id":"couple.n.04"},{"index":5,"sense_key":"couple%1:17:00::","synset_id":"couple.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"couple%1:23:01::","call_ids":["semeval2013.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"couple%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001158,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.48408591595944}
{"item_id":"semeval2013.d001.s012.t002","gold_sense_keys":["play%1:04:05::"],"candidates":[{"index":1,"sense_key":"play%1:10:01::","synset_id":"play.n.01"},{"index":2,"sense_key":"play%1:10:03::","synset_id":"play.n.02"},{"index":3,"sense_key":"play%1:04:05::","synset_id":"play.n.03"},{"index":4,"sense_key":"play%1:04:04::","synset_id":"maneuver.n.03"},{"index":5,"sense_key":"play%1:26:00::","synset_id":"play.n.05"},{"index":6,"sense_key":"play%1:04:08::","synset_id":"play.n.06"},{"index":7,"sense_key":"play%1:04:03::","synset_id":"bid.n.02"},{"index":8,"sense_key":"play%1:04:06::","synset_id":"play.n.08"},{"index":9,"sense_key":"play%1:28:00::","synset_id":"playing_period.n.01"},{"index":10,"sense_key":"play%1:26:01::","synset_id":"free_rein.n.01"},{"index":11,"sense_key":"play%1:11:00::","synset_id":"shimmer.n.01"},{"index":12,"sense_key":"play%1:10:02::","synset_id":"fun.n.02"},{"index":13,"sense_key":"play%1:07:00::","synset_id":"looseness.n.05"},{"index":14,"sense_key":"play%1:04:00::","synset_id":"play.n.14"},{"index":15,"sense_key":"play%1:04:01::","synset_id":"turn.n.03"},{"index":16,"sense_key":"play%1:04:02::","synset_id":"gambling.n.01"},{"index":17,"sense_key":"play%1:04:07::","synset_id":"play.n.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"play%1:04:05::","call_ids":["semeval2013.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"play%1:04:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1252,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.000197,"input_uncached_usd":0.0000626,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.501870499982033}
{"item_id":"semeval2013.d001.s012.t003","gold_sense_keys":["help%1:04:00::"],"candidates":[{"index":1,"sense_key":"help%1:04:00::","synset_id":"aid.n.02"},{"index":2,"sense_key":"help%1:18:00::","synset_id":"assistant.n.01"},{"index":3,"sense_key":"help%1:07:00::","synset_id":"aid.n.01"},{"index":4,"sense_key":"help%1:07:01::","synset_id":"avail.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"help%1:04:00::","call_ids":["semeval2013.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"help%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037189999999999993,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.705525750003289}
{"item_id":"semeval2013.d001.s012.t005","gold_sense_keys":["lead%1:23:00::"],"candidates":[{"index":1,"sense_key":"lead%1:07:02::","synset_id":"lead.n.01"},{"index":2,"sense_key":"lead%1:27:00::","synset_id":"lead.n.02"},{"index":3,"sense_key":"lead%1:09:00::","synset_id":"lead.n.03"},{"index":4,"sense_key":"lead%1:04:01::","synset_id":"lead.n.04"},{"index":5,"sense_key":"lead%1:25:00::","synset_id":"lead.n.05"},{"index":6,"sense_key":"lead%1:10:01::","synset_id":"lead.n.06"},{"index":7,"sense_key":"lead%1:23:00::","synset_id":"lead.n.07"},{"index":8,"sense_key":"lead%1:18:00::","synset_id":"star.n.04"},{"index":9,"sense_key":"lead%1:15:00::","synset_id":"lead.n.09"},{"index":10,"sense_key":"lead%1:10:02::","synset_id":"tip.n.03"},{"index":11,"sense_key":"lead%1:10:00::","synset_id":"lead.n.11"},{"index":12,"sense_key":"lead%1:07:01::","synset_id":"spark_advance.n.01"},{"index":13,"sense_key":"lead%1:06:03::","synset_id":"leash.n.01"},{"index":14,"sense_key":"lead%1:06:02::","synset_id":"lead.n.14"},{"index":15,"sense_key":"lead%1:06:00::","synset_id":"lead.n.15"},{"index":16,"sense_key":"lead%1:06:01::","synset_id":"jumper_cable.n.01"},{"index":17,"sense_key":"lead%1:04:00::","synset_id":"lead.n.17"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"lead%1:23:00::","call_ids":["semeval2013.d001.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"lead%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1177,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00019324999999999998,"input_uncached_usd":0.00005885,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.432151374989189}
{"item_id":"semeval2013.d001.s012.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s012.t006__v1__a1","semeval2013.d001.s012.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1095,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00075875,"input_uncached_usd":0.00005475,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.66758370801108}
{"item_id":"semeval2013.d001.s012.t007","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d001.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1062,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036669999999999997,"input_uncached_usd":0.000053099999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.753968500008341}
{"item_id":"semeval2013.d001.s012.t008","gold_sense_keys":["quarter%1:28:01::"],"candidates":[{"index":1,"sense_key":"quarter%1:23:01::","synset_id":"one-fourth.n.01"},{"index":2,"sense_key":"quarter%1:15:00::","synset_id":"quarter.n.02"},{"index":3,"sense_key":"quarter%1:28:01::","synset_id":"quarter.n.03"},{"index":4,"sense_key":"quarter%1:28:03::","synset_id":"quarter.n.04"},{"index":5,"sense_key":"quarter%1:28:00::","synset_id":"quarter.n.05"},{"index":6,"sense_key":"quarter%1:28:02::","synset_id":"quarter.n.06"},{"index":7,"sense_key":"quarter%1:24:00::","synset_id":"quarter.n.07"},{"index":8,"sense_key":"quarter%1:23:02::","synset_id":"quarter.n.08"},{"index":9,"sense_key":"quarter%1:23:04::","synset_id":"quarter.n.09"},{"index":10,"sense_key":"quarter%1:21:00::","synset_id":"quarter.n.10"},{"index":11,"sense_key":"quarter%1:18:00::","synset_id":"quarter.n.11"},{"index":12,"sense_key":"quarter%1:06:00::","synset_id":"stern.n.01"},{"index":13,"sense_key":"quarter%1:06:01::","synset_id":"quarter.n.13"},{"index":14,"sense_key":"quarter%1:04:00::","synset_id":"quarter.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"quarter%1:28:01::","call_ids":["semeval2013.d001.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"quarter%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00028544999999999997,"input_uncached_usd":0.00004865,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.038711000001058}
{"item_id":"semeval2013.d001.s013.t001","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2013.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001946,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.894595667021349}
{"item_id":"semeval2013.d001.s014.t000","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039039999999999995,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.260953666002024}
{"item_id":"semeval2013.d001.s014.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s014.t001__v1__a1","semeval2013.d001.s014.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1073,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00073205,"input_uncached_usd":0.00005365,"input_cached_usd":0.0,"output_usd":0.0006783999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.982220250938553}
{"item_id":"semeval2013.d001.s014.t002","gold_sense_keys":["line%1:15:01::"],"candidates":[{"index":1,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":2,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":3,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":4,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":5,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":6,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":7,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":8,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":9,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":10,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":11,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":12,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":13,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":14,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":15,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":16,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":17,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":18,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":19,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":20,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":21,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":22,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":23,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":24,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":25,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":26,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":27,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":28,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":29,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":30,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":20,"chosen_sense_key":"line%1:15:01::","call_ids":["semeval2013.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":20,"predicted_sense_key":"line%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1704,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00032199999999999997,"input_uncached_usd":0.0000852,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.348318582982756}
{"item_id":"semeval2013.d001.s015.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d001.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002852,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.260798292001709}
{"item_id":"semeval2013.d001.s015.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005340000000000001,"input_uncached_usd":0.0000412,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.405178125016391}
{"item_id":"semeval2013.d001.s015.t005","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2013.d001.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035384999999999995,"input_uncached_usd":0.000040249999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.403056207986083}
{"item_id":"semeval2013.d001.s015.t006","gold_sense_keys":["season%1:28:02::"],"candidates":[{"index":1,"sense_key":"season%1:28:02::","synset_id":"season.n.01"},{"index":2,"sense_key":"season%1:28:00::","synset_id":"season.n.02"},{"index":3,"sense_key":"season%1:28:01::","synset_id":"season.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"season%1:28:02::","call_ids":["semeval2013.d001.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"season%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005708999999999999,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.503765833971556}
{"item_id":"semeval2013.d002.s000.t000","gold_sense_keys":["wall_street%1:14:00::"],"candidates":[{"index":1,"sense_key":"wall_street%1:15:00::","synset_id":"wall_street.n.01"},{"index":2,"sense_key":"wall_street%1:14:00::","synset_id":"wall_street.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"wall_street%1:15:00::","call_ids":["semeval2013.d002.s000.t000__v1__a1","semeval2013.d002.s000.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"wall_street%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":1441,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006075499999999999,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.0005763999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.137002167000901}
{"item_id":"semeval2013.d002.s000.t002","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":362,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001525,"input_uncached_usd":0.0000181,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.726668208022602}
{"item_id":"semeval2013.d002.s001.t001","gold_sense_keys":["heading%1:15:00::"],"candidates":[{"index":1,"sense_key":"heading%1:10:00::","synset_id":"heading.n.01"},{"index":2,"sense_key":"heading%1:15:00::","synset_id":"bearing.n.02"},{"index":3,"sense_key":"heading%1:06:00::","synset_id":"drift.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"heading%1:15:00::","call_ids":["semeval2013.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"heading%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033339999999999997,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.420172834012192}
{"item_id":"semeval2013.d002.s001.t003","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018404999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.651561999984551}
{"item_id":"semeval2013.d002.s001.t004","gold_sense_keys":["indicator%1:10:00::"],"candidates":[{"index":1,"sense_key":"indicator%1:10:00::","synset_id":"index.n.02"},{"index":2,"sense_key":"indicator%1:10:01::","synset_id":"indicator.n.02"},{"index":3,"sense_key":"indicator%1:06:00::","synset_id":"indicator.n.03"},{"index":4,"sense_key":"indicator%1:27:00::","synset_id":"indicator.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"indicator%1:10:00::","call_ids":["semeval2013.d002.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"indicator%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001816,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.563880249974318}
{"item_id":"semeval2013.d002.s001.t005","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d002.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015375,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.053962333011441}
{"item_id":"semeval2013.d002.s001.t006","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018079999999999998,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.158962208952289}
{"item_id":"semeval2013.d002.s002.t001","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":2,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":4,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":5,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":6,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":7,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"},{"index":8,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":9,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":10,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":11,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":12,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":13,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"figure%1:07:01::","call_ids":["semeval2013.d002.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"figure%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":828,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045739999999999995,"input_uncached_usd":0.0000414,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.650786959042307}
{"item_id":"semeval2013.d002.s002.t006","gold_sense_keys":["predominance%1:26:00::"],"candidates":[{"index":1,"sense_key":"predominance%1:26:00::","synset_id":"predominance.n.01"},{"index":2,"sense_key":"predominance%1:07:00::","synset_id":"predomination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"predominance%1:26:00::","call_ids":["semeval2013.d002.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"predominance%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001535,"input_uncached_usd":0.0000191,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1054148330003954}
{"item_id":"semeval2013.d002.s004.t000","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023945,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.663221709022764}
{"item_id":"semeval2013.d002.s005.t002","gold_sense_keys":["volume%1:07:03::"],"candidates":[{"index":1,"sense_key":"volume%1:23:00::","synset_id":"volume.n.01"},{"index":2,"sense_key":"volume%1:07:03::","synset_id":"bulk.n.02"},{"index":3,"sense_key":"volume%1:06:00::","synset_id":"book.n.02"},{"index":4,"sense_key":"volume%1:10:00::","synset_id":"volume.n.04"},{"index":5,"sense_key":"volume%1:23:01::","synset_id":"volume.n.05"},{"index":6,"sense_key":"volume%1:07:02::","synset_id":"volume.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"volume%1:23:01::","call_ids":["semeval2013.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"volume%1:23:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024405,"input_uncached_usd":0.000032449999999999996,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.327668417012319}
{"item_id":"semeval2013.d002.s005.t003","gold_sense_keys":["case%1:09:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d002.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1250,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003505,"input_uncached_usd":0.0000625,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.400337291997857}
{"item_id":"semeval2013.d002.s005.t004","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":982,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003627,"input_uncached_usd":0.0000491,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.135579665948171}
{"item_id":"semeval2013.d002.s005.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":583,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013795,"input_uncached_usd":0.000029149999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3982773749739863}
{"item_id":"semeval2013.d002.s006.t000","gold_sense_keys":["trend%1:15:02::"],"candidates":[{"index":1,"sense_key":"trend%1:15:02::","synset_id":"tendency.n.04"},{"index":2,"sense_key":"trend%1:15:01::","synset_id":"course.n.03"},{"index":3,"sense_key":"trend%1:09:00::","synset_id":"drift.n.05"},{"index":4,"sense_key":"trend%1:09:02::","synset_id":"vogue.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trend%1:15:02::","call_ids":["semeval2013.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trend%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029225,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.410968957992736}
{"item_id":"semeval2013.d002.s006.t001","gold_sense_keys":["publication%1:04:02::"],"candidates":[{"index":1,"sense_key":"publication%1:10:00::","synset_id":"publication.n.01"},{"index":2,"sense_key":"publication%1:04:01::","synset_id":"issue.n.11"},{"index":3,"sense_key":"publication%1:04:02::","synset_id":"publication.n.03"},{"index":4,"sense_key":"publication%1:04:00::","synset_id":"publication.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"publication%1:04:02::","call_ids":["semeval2013.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"publication%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013489999999999997,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9559574580052868}
{"item_id":"semeval2013.d002.s006.t002","gold_sense_keys":["indicator%1:10:00::"],"candidates":[{"index":1,"sense_key":"indicator%1:10:00::","synset_id":"index.n.02"},{"index":2,"sense_key":"indicator%1:10:01::","synset_id":"indicator.n.02"},{"index":3,"sense_key":"indicator%1:06:00::","synset_id":"indicator.n.03"},{"index":4,"sense_key":"indicator%1:27:00::","synset_id":"indicator.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"indicator%1:10:00::","call_ids":["semeval2013.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"indicator%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003164,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.235333667020313}
{"item_id":"semeval2013.d002.s006.t003","gold_sense_keys":["market%1:14:01::"],"candidates":[{"index":1,"sense_key":"market%1:04:00::","synset_id":"market.n.01"},{"index":2,"sense_key":"market%1:14:00::","synset_id":"market.n.02"},{"index":3,"sense_key":"market%1:06:00::","synset_id":"grocery_store.n.01"},{"index":4,"sense_key":"market%1:14:01::","synset_id":"market.n.04"},{"index":5,"sense_key":"market%1:06:01::","synset_id":"marketplace.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"market%1:14:01::","call_ids":["semeval2013.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"market%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001913,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9932553750113584}
{"item_id":"semeval2013.d002.s006.t005","gold_sense_keys":["relation%1:03:00::"],"candidates":[{"index":1,"sense_key":"relation%1:03:00::","synset_id":"relation.n.01"},{"index":2,"sense_key":"relation%1:04:01::","synset_id":"sexual_intercourse.n.01"},{"index":3,"sense_key":"relation%1:18:00::","synset_id":"relative.n.01"},{"index":4,"sense_key":"relation%1:10:01::","synset_id":"relation.n.04"},{"index":5,"sense_key":"relation%1:09:00::","synset_id":"relation_back.n.01"},{"index":6,"sense_key":"relation%1:04:00::","synset_id":"relation.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relation%1:03:00::","call_ids":["semeval2013.d002.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relation%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005027,"input_uncached_usd":0.0000351,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.818391333974432}
{"item_id":"semeval2013.d002.s006.t007","gold_sense_keys":["recovery%1:11:00::"],"candidates":[{"index":1,"sense_key":"recovery%1:11:00::","synset_id":"recovery.n.01"},{"index":2,"sense_key":"recovery%1:22:00::","synset_id":"convalescence.n.01"},{"index":3,"sense_key":"recovery%1:04:00::","synset_id":"recovery.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recovery%1:11:00::","call_ids":["semeval2013.d002.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recovery%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046764999999999996,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.991906750015914}
{"item_id":"semeval2013.d002.s006.t008","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":2,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"},{"index":3,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026205,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.663212332990952}
{"item_id":"semeval2013.d002.s007.t000","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007235000000000001,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.645063124946319}
{"item_id":"semeval2013.d002.s007.t003","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2013.d002.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005932000000000001,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.849746708001476}
{"item_id":"semeval2013.d002.s007.t004","gold_sense_keys":["row%1:07:00::"],"candidates":[{"index":1,"sense_key":"row%1:14:00::","synset_id":"row.n.01"},{"index":2,"sense_key":"row%1:10:00::","synset_id":"quarrel.n.01"},{"index":3,"sense_key":"row%1:17:00::","synset_id":"row.n.03"},{"index":4,"sense_key":"row%1:06:00::","synset_id":"course.n.08"},{"index":5,"sense_key":"row%1:14:01::","synset_id":"row.n.05"},{"index":6,"sense_key":"row%1:07:00::","synset_id":"row.n.06"},{"index":7,"sense_key":"row%1:04:00::","synset_id":"rowing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"row%1:07:00::","call_ids":["semeval2013.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"row%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019214999999999998,"input_uncached_usd":0.000032149999999999995,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.536799458961468}
{"item_id":"semeval2013.d002.s008.t000","gold_sense_keys":["confidence%1:12:00::","confidence%1:26:02::"],"candidates":[{"index":1,"sense_key":"confidence%1:09:00::","synset_id":"assurance.n.01"},{"index":2,"sense_key":"confidence%1:12:00::","synset_id":"confidence.n.02"},{"index":3,"sense_key":"confidence%1:26:02::","synset_id":"confidence.n.03"},{"index":4,"sense_key":"confidence%1:26:03::","synset_id":"confidence.n.04"},{"index":5,"sense_key":"confidence%1:10:00::","synset_id":"confidence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"confidence%1:26:02::","call_ids":["semeval2013.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"confidence%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002677,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.664761499967426}
{"item_id":"semeval2013.d002.s008.t003","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d002.s008.t003__v1__a1","semeval2013.d002.s008.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":979,"cached_input_tokens":0,"output_tokens":1441,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00062535,"input_uncached_usd":0.00004895,"input_cached_usd":0.0,"output_usd":0.0005763999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.566151000035461}
{"item_id":"semeval2013.d002.s008.t004","gold_sense_keys":["estimate%1:09:00::"],"candidates":[{"index":1,"sense_key":"estimate%1:09:00::","synset_id":"estimate.n.01"},{"index":2,"sense_key":"estimate%1:04:01::","synset_id":"estimate.n.02"},{"index":3,"sense_key":"estimate%1:10:00::","synset_id":"appraisal.n.02"},{"index":4,"sense_key":"estimate%1:10:01::","synset_id":"estimate.n.04"},{"index":5,"sense_key":"estimate%1:09:01::","synset_id":"estimate.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"estimate%1:09:00::","call_ids":["semeval2013.d002.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"estimate%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031865,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.600416375033092}
{"item_id":"semeval2013.d002.s008.t005","gold_sense_keys":["index%1:10:01::"],"candidates":[{"index":1,"sense_key":"index%1:24:00::","synset_id":"index.n.01"},{"index":2,"sense_key":"index%1:10:01::","synset_id":"index.n.02"},{"index":3,"sense_key":"index%1:10:02::","synset_id":"exponent.n.03"},{"index":4,"sense_key":"index%1:10:00::","synset_id":"index.n.04"},{"index":5,"sense_key":"index%1:08:00::","synset_id":"index.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"index%1:10:01::","call_ids":["semeval2013.d002.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"index%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002657,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4149762080051005}
{"item_id":"semeval2013.d002.s009.t000","gold_sense_keys":["index%1:10:01::"],"candidates":[{"index":1,"sense_key":"index%1:24:00::","synset_id":"index.n.01"},{"index":2,"sense_key":"index%1:10:01::","synset_id":"index.n.02"},{"index":3,"sense_key":"index%1:10:02::","synset_id":"exponent.n.03"},{"index":4,"sense_key":"index%1:10:00::","synset_id":"index.n.04"},{"index":5,"sense_key":"index%1:08:00::","synset_id":"index.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"index%1:10:01::","call_ids":["semeval2013.d002.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"index%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059725,"input_uncached_usd":0.000027249999999999998,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.568249041039962}
{"item_id":"semeval2013.d002.s009.t001","gold_sense_keys":["rise%1:04:01::","rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":2,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":3,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":4,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":5,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":6,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":9,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":10,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"rise%1:04:01::","call_ids":["semeval2013.d002.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"rise%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004291,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.421353708021343}
{"item_id":"semeval2013.d002.s009.t002","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026264999999999996,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.736825291998684}
{"item_id":"semeval2013.d002.s009.t003","gold_sense_keys":["high%1:07:00::"],"candidates":[{"index":1,"sense_key":"high%1:07:00::","synset_id":"high.n.01"},{"index":2,"sense_key":"high%1:26:00::","synset_id":"high.n.02"},{"index":3,"sense_key":"high%1:26:02::","synset_id":"high.n.03"},{"index":4,"sense_key":"high%1:26:01::","synset_id":"high.n.04"},{"index":5,"sense_key":"high%1:15:00::","synset_id":"high.n.05"},{"index":6,"sense_key":"high%1:14:00::","synset_id":"senior_high_school.n.01"},{"index":7,"sense_key":"high%1:06:00::","synset_id":"high_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"high%1:26:02::","call_ids":["semeval2013.d002.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"high%1:26:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004988999999999999,"input_uncached_usd":0.0000317,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.5300386659801}
{"item_id":"semeval2013.d002.s009.t005","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d002.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.000578,"input_uncached_usd":0.000034,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.195051332993899}
{"item_id":"semeval2013.d002.s009.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d002.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052135,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.4955841670162044}
{"item_id":"semeval2013.d002.s009.t007","gold_sense_keys":["situation%1:26:00::"],"candidates":[{"index":1,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"},{"index":2,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":3,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":4,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":5,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"situation%1:26:00::","call_ids":["semeval2013.d002.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"situation%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047204999999999995,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.659513791964855}
{"item_id":"semeval2013.d002.s010.t001","gold_sense_keys":["security%1:21:04::"],"candidates":[{"index":1,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":2,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":3,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":4,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":5,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":6,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":7,"sense_key":"security%1:10:00::","synset_id":"security.n.07"},{"index":8,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":9,"sense_key":"security%1:04:00::","synset_id":"security.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"security%1:21:04::","call_ids":["semeval2013.d002.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"security%1:21:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045315,"input_uncached_usd":0.000037149999999999995,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.185284540988505}
{"item_id":"semeval2013.d002.s010.t002","gold_sense_keys":["ground%1:15:00::"],"candidates":[{"index":1,"sense_key":"ground%1:17:00::","synset_id":"land.n.04"},{"index":2,"sense_key":"ground%1:16:00::","synset_id":"reason.n.01"},{"index":3,"sense_key":"ground%1:27:00::","synset_id":"earth.n.02"},{"index":4,"sense_key":"ground%1:24:00::","synset_id":"footing.n.02"},{"index":5,"sense_key":"ground%1:15:00::","synset_id":"ground.n.05"},{"index":6,"sense_key":"ground%1:09:00::","synset_id":"background.n.02"},{"index":7,"sense_key":"ground%1:17:01::","synset_id":"land.n.02"},{"index":8,"sense_key":"ground%1:09:01::","synset_id":"ground.n.08"},{"index":9,"sense_key":"ground%1:06:02::","synset_id":"ground.n.09"},{"index":10,"sense_key":"ground%1:06:01::","synset_id":"ground.n.10"},{"index":11,"sense_key":"ground%1:06:00::","synset_id":"flat_coat.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"ground%1:15:00::","call_ids":["semeval2013.d002.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"ground%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":871,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048554999999999996,"input_uncached_usd":0.000043549999999999995,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.662130665965378}
{"item_id":"semeval2013.d002.s012.t001","gold_sense_keys":["bond%1:21:02::"],"candidates":[{"index":1,"sense_key":"bond%1:19:00::","synset_id":"chemical_bond.n.01"},{"index":2,"sense_key":"bond%1:21:02::","synset_id":"bond.n.02"},{"index":3,"sense_key":"bond%1:24:00::","synset_id":"alliance.n.02"},{"index":4,"sense_key":"bond%1:21:03::","synset_id":"bail.n.01"},{"index":5,"sense_key":"bond%1:06:00::","synset_id":"shackle.n.01"},{"index":6,"sense_key":"bond%1:06:01::","synset_id":"attachment.n.04"},{"index":7,"sense_key":"bond%1:27:00::","synset_id":"bond.n.07"},{"index":8,"sense_key":"bond%1:18:00::","synset_id":"bond.n.08"},{"index":9,"sense_key":"bond%1:18:01::","synset_id":"bond.n.09"},{"index":10,"sense_key":"bond%1:07:00::","synset_id":"adhesiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bond%1:21:02::","call_ids":["semeval2013.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bond%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025164999999999996,"input_uncached_usd":0.00004045,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.044164208986331}
{"item_id":"semeval2013.d002.s012.t002","gold_sense_keys":["yield%1:21:00::"],"candidates":[{"index":1,"sense_key":"yield%1:04:00::","synset_id":"output.n.02"},{"index":2,"sense_key":"yield%1:21:00::","synset_id":"return.n.06"},{"index":3,"sense_key":"yield%1:06:00::","synset_id":"yield.n.03"},{"index":4,"sense_key":"yield%1:23:00::","synset_id":"output.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"yield%1:06:00::","call_ids":["semeval2013.d002.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"yield%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006966499999999999,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.174880916951224}
{"item_id":"semeval2013.d002.s012.t004","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"evening%1:28:00::","call_ids":["semeval2013.d002.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"evening%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003366,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.10735545802163}
{"item_id":"semeval2013.d002.s012.t005","gold_sense_keys":["bond%1:21:02::"],"candidates":[{"index":1,"sense_key":"bond%1:19:00::","synset_id":"chemical_bond.n.01"},{"index":2,"sense_key":"bond%1:21:02::","synset_id":"bond.n.02"},{"index":3,"sense_key":"bond%1:24:00::","synset_id":"alliance.n.02"},{"index":4,"sense_key":"bond%1:21:03::","synset_id":"bail.n.01"},{"index":5,"sense_key":"bond%1:06:00::","synset_id":"shackle.n.01"},{"index":6,"sense_key":"bond%1:06:01::","synset_id":"attachment.n.04"},{"index":7,"sense_key":"bond%1:27:00::","synset_id":"bond.n.07"},{"index":8,"sense_key":"bond%1:18:00::","synset_id":"bond.n.08"},{"index":9,"sense_key":"bond%1:18:01::","synset_id":"bond.n.09"},{"index":10,"sense_key":"bond%1:07:00::","synset_id":"adhesiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bond%1:21:02::","call_ids":["semeval2013.d002.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bond%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025164999999999996,"input_uncached_usd":0.00004045,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.119261291984003}
{"item_id":"semeval2013.d002.s012.t006","gold_sense_keys":["yield%1:21:00::"],"candidates":[{"index":1,"sense_key":"yield%1:04:00::","synset_id":"output.n.02"},{"index":2,"sense_key":"yield%1:21:00::","synset_id":"return.n.06"},{"index":3,"sense_key":"yield%1:06:00::","synset_id":"yield.n.03"},{"index":4,"sense_key":"yield%1:23:00::","synset_id":"output.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"yield%1:21:00::","call_ids":["semeval2013.d002.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"yield%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044064999999999995,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.103740959020797}
{"item_id":"semeval2013.d002.s012.t007","gold_sense_keys":["evening%1:28:00::"],"candidates":[{"index":1,"sense_key":"evening%1:28:00::","synset_id":"evening.n.01"},{"index":2,"sense_key":"evening%1:28:01::","synset_id":"evening.n.02"},{"index":3,"sense_key":"evening%1:28:02::","synset_id":"evening.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evening%1:28:02::","call_ids":["semeval2013.d002.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evening%1:28:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031105,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.398355041979812}
{"item_id":"semeval2013.d003.s000.t000","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003173,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.059891624958254}
{"item_id":"semeval2013.d003.s001.t000","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"concern%1:09:01::","call_ids":["semeval2013.d003.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"concern%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033725,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.764851375017315}
{"item_id":"semeval2013.d003.s001.t001","gold_sense_keys":["violence%1:26:00::"],"candidates":[{"index":1,"sense_key":"violence%1:04:01::","synset_id":"violence.n.01"},{"index":2,"sense_key":"violence%1:07:00::","synset_id":"ferocity.n.01"},{"index":3,"sense_key":"violence%1:26:00::","synset_id":"violence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"violence%1:26:00::","call_ids":["semeval2013.d003.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"violence%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":377,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020444999999999998,"input_uncached_usd":0.00001885,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.22310737502994}
{"item_id":"semeval2013.d003.s001.t002","gold_sense_keys":["instability%1:07:00::","instability%1:07:01::","instability%1:26:00::"],"candidates":[{"index":1,"sense_key":"instability%1:26:00::","synset_id":"instability.n.01"},{"index":2,"sense_key":"instability%1:07:01::","synset_id":"instability.n.02"},{"index":3,"sense_key":"instability%1:26:01::","synset_id":"imbalance.n.01"},{"index":4,"sense_key":"instability%1:07:00::","synset_id":"instability.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"instability%1:26:01::","call_ids":["semeval2013.d003.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"instability%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040835,"input_uncached_usd":0.00001795,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.610025166010018}
{"item_id":"semeval2013.d003.s001.t003","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d003.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036009999999999997,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.848167875024956}
{"item_id":"semeval2013.d003.s001.t006","gold_sense_keys":["auction%1:04:00::"],"candidates":[{"index":1,"sense_key":"auction%1:04:01::","synset_id":"auction.n.01"},{"index":2,"sense_key":"auction%1:04:00::","synset_id":"auction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"auction%1:04:00::","call_ids":["semeval2013.d003.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"auction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":318,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004067,"input_uncached_usd":0.0000159,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.080291583028156}
{"item_id":"semeval2013.d003.s002.t000","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032045,"input_uncached_usd":0.000032449999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.542698874953203}
{"item_id":"semeval2013.d003.s002.t002","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d003.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028375,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3698155409656465}
{"item_id":"semeval2013.d003.s002.t003","gold_sense_keys":["russia%1:15:04::"],"candidates":[{"index":1,"sense_key":"russia%1:15:00::","synset_id":"soviet_union.n.01"},{"index":2,"sense_key":"russia%1:15:01::","synset_id":"soviet_russia.n.01"},{"index":3,"sense_key":"russia%1:15:02::","synset_id":"russia.n.03"},{"index":4,"sense_key":"russia%1:15:04::","synset_id":"russia.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"russia%1:15:04::","call_ids":["semeval2013.d003.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"russia%1:15:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001333,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2295146660180762}
{"item_id":"semeval2013.d003.s002.t004","gold_sense_keys":["oil%1:27:01::"],"candidates":[{"index":1,"sense_key":"oil%1:27:00::","synset_id":"oil.n.01"},{"index":2,"sense_key":"oil%1:06:01::","synset_id":"oil.n.02"},{"index":3,"sense_key":"oil%1:27:01::","synset_id":"petroleum.n.01"},{"index":4,"sense_key":"oil%1:13:00::","synset_id":"vegetable_oil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"oil%1:27:01::","call_ids":["semeval2013.d003.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"oil%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007745,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0504650000366382}
{"item_id":"semeval2013.d003.s002.t006","gold_sense_keys":["role%1:04:00::"],"candidates":[{"index":1,"sense_key":"role%1:04:00::","synset_id":"function.n.03"},{"index":2,"sense_key":"role%1:09:00::","synset_id":"character.n.04"},{"index":3,"sense_key":"role%1:07:00::","synset_id":"function.n.02"},{"index":4,"sense_key":"role%1:04:01::","synset_id":"role.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"role%1:04:00::","call_ids":["semeval2013.d003.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"role%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046389999999999995,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.440541749994736}
{"item_id":"semeval2013.d003.s002.t009","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d003.s002.t009__v1__a1","semeval2013.d003.s002.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":865,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038185,"input_uncached_usd":0.000035850000000000004,"input_cached_usd":0.0,"output_usd":0.00034599999999999995,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.933791416988242}
{"item_id":"semeval2013.d003.s002.t010","gold_sense_keys":["war%1:04:00::","war%1:26:00::"],"candidates":[{"index":1,"sense_key":"war%1:04:00::","synset_id":"war.n.01"},{"index":2,"sense_key":"war%1:26:00::","synset_id":"war.n.02"},{"index":3,"sense_key":"war%1:04:02::","synset_id":"war.n.03"},{"index":4,"sense_key":"war%1:04:01::","synset_id":"war.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"war%1:04:00::","call_ids":["semeval2013.d003.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"war%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018184999999999997,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.351434624986723}
{"item_id":"semeval2013.d003.s002.t011","gold_sense_keys":["sanction%1:04:02::"],"candidates":[{"index":1,"sense_key":"sanction%1:10:00::","synset_id":"sanction.n.01"},{"index":2,"sense_key":"sanction%1:04:02::","synset_id":"sanction.n.02"},{"index":3,"sense_key":"sanction%1:07:00::","synset_id":"authority.n.06"},{"index":4,"sense_key":"sanction%1:04:01::","synset_id":"sanction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sanction%1:04:02::","call_ids":["semeval2013.d003.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sanction%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003082,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.157781291985884}
{"item_id":"semeval2013.d003.s003.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011429999999999999,"input_uncached_usd":0.0000311,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3346858330187388}
{"item_id":"semeval2013.d003.s003.t001","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":2,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":3,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":4,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"},{"index":5,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d003.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020979999999999998,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.183909000013955}
{"item_id":"semeval2013.d003.s003.t002","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004937499999999999,"input_uncached_usd":0.00005215,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.2879796249908395}
{"item_id":"semeval2013.d003.s004.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037074999999999993,"input_uncached_usd":0.00003155,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.321236541029066}
{"item_id":"semeval2013.d003.s004.t001","gold_sense_keys":["auction%1:04:00::"],"candidates":[{"index":1,"sense_key":"auction%1:04:01::","synset_id":"auction.n.01"},{"index":2,"sense_key":"auction%1:04:00::","synset_id":"auction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"auction%1:04:00::","call_ids":["semeval2013.d003.s004.t001__v1__a1","semeval2013.d003.s004.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"auction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":763,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043495,"input_uncached_usd":0.00003814999999999999,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.650503249955364}
{"item_id":"semeval2013.d003.s005.t000","gold_sense_keys":["bid%1:10:00::"],"candidates":[{"index":1,"sense_key":"bid%1:10:03::","synset_id":"command.n.01"},{"index":2,"sense_key":"bid%1:04:00::","synset_id":"bid.n.02"},{"index":3,"sense_key":"bid%1:10:00::","synset_id":"bid.n.03"},{"index":4,"sense_key":"bid%1:10:01::","synset_id":"bid.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bid%1:10:00::","call_ids":["semeval2013.d003.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bid%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000793,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4539848749991506}
{"item_id":"semeval2013.d003.s006.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006019500000000001,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.18928049999522}
{"item_id":"semeval2013.d003.s006.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034809999999999995,"input_uncached_usd":0.0000345,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.475842250045389}
{"item_id":"semeval2013.d003.s006.t002","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046944999999999995,"input_uncached_usd":0.00005345,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.3374647500459105}
{"item_id":"semeval2013.d003.s007.t000","gold_sense_keys":["representation%1:04:01::"],"candidates":[{"index":1,"sense_key":"representation%1:09:00::","synset_id":"representation.n.01"},{"index":2,"sense_key":"representation%1:06:00::","synset_id":"representation.n.02"},{"index":3,"sense_key":"representation%1:04:01::","synset_id":"representation.n.03"},{"index":4,"sense_key":"representation%1:26:00::","synset_id":"representation.n.04"},{"index":5,"sense_key":"representation%1:14:00::","synset_id":"representation.n.05"},{"index":6,"sense_key":"representation%1:10:00::","synset_id":"representation.n.06"},{"index":7,"sense_key":"representation%1:10:01::","synset_id":"theatrical_performance.n.01"},{"index":8,"sense_key":"representation%1:10:02::","synset_id":"representation.n.08"},{"index":9,"sense_key":"representation%1:07:00::","synset_id":"representation.n.09"},{"index":10,"sense_key":"representation%1:04:00::","synset_id":"representation.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"representation%1:06:00::","call_ids":["semeval2013.d003.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"representation%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007310000000000001,"input_uncached_usd":0.0000334,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.8154235840193}
{"item_id":"semeval2013.d003.s007.t001","gold_sense_keys":["oil%1:27:01::"],"candidates":[{"index":1,"sense_key":"oil%1:27:00::","synset_id":"oil.n.01"},{"index":2,"sense_key":"oil%1:06:01::","synset_id":"oil.n.02"},{"index":3,"sense_key":"oil%1:27:01::","synset_id":"petroleum.n.01"},{"index":4,"sense_key":"oil%1:13:00::","synset_id":"vegetable_oil.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"oil%1:27:01::","call_ids":["semeval2013.d003.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"oil%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025949999999999997,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7454099580063485}
{"item_id":"semeval2013.d003.s007.t002","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":2,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":3,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":4,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":6,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":7,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d003.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026455,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.94081058294978}
{"item_id":"semeval2013.d003.s007.t005","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":2,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"},{"index":3,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d003.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028274999999999996,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.764997749996837}
{"item_id":"semeval2013.d003.s008.t001","gold_sense_keys":["door%1:07:00::"],"candidates":[{"index":1,"sense_key":"door%1:06:00::","synset_id":"door.n.01"},{"index":2,"sense_key":"door%1:06:01::","synset_id":"doorway.n.01"},{"index":3,"sense_key":"door%1:07:00::","synset_id":"door.n.03"},{"index":4,"sense_key":"door%1:06:04::","synset_id":"door.n.04"},{"index":5,"sense_key":"door%1:06:03::","synset_id":"door.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"door%1:07:00::","call_ids":["semeval2013.d003.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"door%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018629999999999997,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.064599042001646}
{"item_id":"semeval2013.d003.s008.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d003.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020945,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3164947499753907}
{"item_id":"semeval2013.d003.s008.t003","gold_sense_keys":["war%1:04:00::","war%1:26:00::"],"candidates":[{"index":1,"sense_key":"war%1:04:00::","synset_id":"war.n.01"},{"index":2,"sense_key":"war%1:26:00::","synset_id":"war.n.02"},{"index":3,"sense_key":"war%1:04:02::","synset_id":"war.n.03"},{"index":4,"sense_key":"war%1:04:01::","synset_id":"war.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"war%1:04:00::","call_ids":["semeval2013.d003.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"war%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018314999999999998,"input_uncached_usd":0.00002315,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6412071250379086}
{"item_id":"semeval2013.d003.s008.t004","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035919999999999995,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7819348750053905}
{"item_id":"semeval2013.d003.s008.t005","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016639999999999998,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.56276387500111}
{"item_id":"semeval2013.d003.s008.t007","gold_sense_keys":["editor%1:18:00::"],"candidates":[{"index":1,"sense_key":"editor%1:18:00::","synset_id":"editor.n.01"},{"index":2,"sense_key":"editor%1:10:00::","synset_id":"editor_program.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"editor%1:18:00::","call_ids":["semeval2013.d003.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"editor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033224999999999997,"input_uncached_usd":0.00001865,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.480136832979042}
{"item_id":"semeval2013.d003.s008.t008","gold_sense_keys":["news%1:10:00::","news%1:10:01::"],"candidates":[{"index":1,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":2,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":3,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":4,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":5,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d003.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"news%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007739,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.29235941695515}
{"item_id":"semeval2013.d003.s008.t009","gold_sense_keys":["outlet%1:06:01::"],"candidates":[{"index":1,"sense_key":"outlet%1:06:01::","synset_id":"mercantile_establishment.n.01"},{"index":2,"sense_key":"outlet%1:06:02::","synset_id":"wall_socket.n.01"},{"index":3,"sense_key":"outlet%1:06:00::","synset_id":"exit.n.01"},{"index":4,"sense_key":"outlet%1:04:00::","synset_id":"release.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"outlet%1:06:00::","call_ids":["semeval2013.d003.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"outlet%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064445,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.991127667017281}
{"item_id":"semeval2013.d003.s010.t000","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"concern%1:12:00::","call_ids":["semeval2013.d003.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"concern%1:12:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007010499999999999,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.055213542014826}
{"item_id":"semeval2013.d003.s010.t001","gold_sense_keys":["security%1:04:00::","security%1:26:00::"],"candidates":[{"index":1,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":2,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":3,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":4,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":5,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":6,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":7,"sense_key":"security%1:10:00::","synset_id":"security.n.07"},{"index":8,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":9,"sense_key":"security%1:04:00::","synset_id":"security.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"security%1:26:00::","call_ids":["semeval2013.d003.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"security%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058075,"input_uncached_usd":0.00003675,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.730297792004421}
{"item_id":"semeval2013.d003.s010.t002","gold_sense_keys":["bombing%1:04:01::"],"candidates":[{"index":1,"sense_key":"bombing%1:04:00::","synset_id":"bombing.n.01"},{"index":2,"sense_key":"bombing%1:04:01::","synset_id":"bombing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bombing%1:04:00::","call_ids":["semeval2013.d003.s010.t002__v1__a1","semeval2013.d003.s010.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bombing%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":859,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00061895,"input_uncached_usd":0.00004295,"input_cached_usd":0.0,"output_usd":0.0005759999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.522401665977668}
{"item_id":"semeval2013.d003.s010.t004","gold_sense_keys":["instability%1:07:00::","instability%1:07:01::","instability%1:26:00::"],"candidates":[{"index":1,"sense_key":"instability%1:26:00::","synset_id":"instability.n.01"},{"index":2,"sense_key":"instability%1:07:01::","synset_id":"instability.n.02"},{"index":3,"sense_key":"instability%1:26:01::","synset_id":"imbalance.n.01"},{"index":4,"sense_key":"instability%1:07:00::","synset_id":"instability.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"instability%1:26:00::","call_ids":["semeval2013.d003.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"instability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005166,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.036932874994818}
{"item_id":"semeval2013.d003.s010.t005","gold_sense_keys":["u.s.%1:14:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006696,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.105522333004046}
{"item_id":"semeval2013.d003.s010.t009","gold_sense_keys":["world%1:17:00::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"world%1:17:00::","call_ids":["semeval2013.d003.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"world%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":742,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003251,"input_uncached_usd":0.0000371,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.10821787500754}
{"item_id":"semeval2013.d003.s010.t011","gold_sense_keys":["analyst%1:18:00::","analyst%1:18:02::"],"candidates":[{"index":1,"sense_key":"analyst%1:18:00::","synset_id":"analyst.n.01"},{"index":2,"sense_key":"analyst%1:18:02::","synset_id":"analyst.n.02"},{"index":3,"sense_key":"analyst%1:18:01::","synset_id":"analyst.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"analyst%1:18:02::","call_ids":["semeval2013.d003.s010.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"analyst%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006429000000000001,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.921144457999617}
{"item_id":"semeval2013.d003.s011.t000","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"us%1:15:00::","call_ids":["semeval2013.d003.s011.t000__v1__a1","semeval2013.d003.s011.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"us%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":3168,"reasoning_output_tokens":3136},"cost":{"currency":"USD","total_usd":0.0013170500000000002,"input_uncached_usd":0.00004984999999999999,"input_cached_usd":0.0,"output_usd":0.0012672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":21.856859167048242}
{"item_id":"semeval2013.d003.s011.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d003.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1218,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005537000000000001,"input_uncached_usd":0.000060899999999999996,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.279064624977764}
{"item_id":"semeval2013.d003.s011.t005","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"government%1:09:00::","call_ids":["semeval2013.d003.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"government%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006220500000000001,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.844190291012637}
{"item_id":"semeval2013.d003.s011.t006","gold_sense_keys":["energy%1:19:01::"],"candidates":[{"index":1,"sense_key":"energy%1:19:00::","synset_id":"energy.n.01"},{"index":2,"sense_key":"energy%1:07:00::","synset_id":"energy.n.02"},{"index":3,"sense_key":"energy%1:07:02::","synset_id":"energy.n.03"},{"index":4,"sense_key":"energy%1:07:01::","synset_id":"energy.n.04"},{"index":5,"sense_key":"energy%1:26:00::","synset_id":"energy.n.05"},{"index":6,"sense_key":"energy%1:19:01::","synset_id":"energy.n.06"},{"index":7,"sense_key":"energy%1:14:00::","synset_id":"department_of_energy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"energy%1:19:01::","call_ids":["semeval2013.d003.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"energy%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":685,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029665,"input_uncached_usd":0.00003425,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.464053084026091}
{"item_id":"semeval2013.d003.s012.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003759,"input_uncached_usd":0.0000367,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.156316666980274}
{"item_id":"semeval2013.d003.s012.t001","gold_sense_keys":["deal%1:10:00::"],"candidates":[{"index":1,"sense_key":"deal%1:04:02::","synset_id":"deal.n.01"},{"index":2,"sense_key":"deal%1:10:00::","synset_id":"bargain.n.01"},{"index":3,"sense_key":"deal%1:23:00::","synset_id":"batch.n.02"},{"index":4,"sense_key":"deal%1:27:00::","synset_id":"deal.n.04"},{"index":5,"sense_key":"deal%1:27:01::","synset_id":"softwood.n.01"},{"index":6,"sense_key":"deal%1:14:00::","synset_id":"hand.n.06"},{"index":7,"sense_key":"deal%1:11:00::","synset_id":"deal.n.07"},{"index":8,"sense_key":"deal%1:04:00::","synset_id":"deal.n.08"},{"index":9,"sense_key":"deal%1:04:01::","synset_id":"deal.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"deal%1:10:00::","call_ids":["semeval2013.d003.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"deal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032685,"input_uncached_usd":0.000038849999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.293420000001788}
{"item_id":"semeval2013.d003.s012.t002","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":3,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":4,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2013.d003.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010779999999999999,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.169879082997795}
{"item_id":"semeval2013.d003.s013.t000","gold_sense_keys":["u.s.%1:15:00::"],"candidates":[{"index":1,"sense_key":"u.s.%1:14:00::","synset_id":"united_states_government.n.01"},{"index":2,"sense_key":"us%1:15:00::","synset_id":"united_states.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"u.s.%1:14:00::","call_ids":["semeval2013.d003.s013.t000__v1__a1","semeval2013.d003.s013.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"u.s.%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1059,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007569499999999999,"input_uncached_usd":0.00005295,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.797470332996454}
{"item_id":"semeval2013.d003.s013.t002","gold_sense_keys":["tie%1:26:01::"],"candidates":[{"index":1,"sense_key":"tie%1:06:01::","synset_id":"necktie.n.01"},{"index":2,"sense_key":"tie%1:26:01::","synset_id":"affiliation.n.01"},{"index":3,"sense_key":"tie%1:26:00::","synset_id":"tie.n.03"},{"index":4,"sense_key":"tie%1:06:02::","synset_id":"tie.n.04"},{"index":5,"sense_key":"tie%1:06:04::","synset_id":"link.n.02"},{"index":6,"sense_key":"tie%1:11:00::","synset_id":"draw.n.03"},{"index":7,"sense_key":"tie%1:10:00::","synset_id":"tie.n.07"},{"index":8,"sense_key":"tie%1:06:03::","synset_id":"tie.n.08"},{"index":9,"sense_key":"tie%1:06:00::","synset_id":"tie.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"tie%1:26:01::","call_ids":["semeval2013.d003.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"tie%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":838,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035549999999999997,"input_uncached_usd":0.000041899999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.7457004159805365}
{"item_id":"semeval2013.d003.s013.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d003.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003171,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.870383917004801}
{"item_id":"semeval2013.d003.s014.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027275,"input_uncached_usd":0.00003595,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.503562708967365}
{"item_id":"semeval2013.d003.s014.t001","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"},{"index":2,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":3,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015834999999999997,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6947464999975637}
{"item_id":"semeval2013.d003.s015.t000","gold_sense_keys":["bid%2:40:00::"],"candidates":[{"index":1,"sense_key":"bid%2:40:00::","synset_id":"offer.v.07"},{"index":2,"sense_key":"bid%2:32:01::","synset_id":"wish.v.06"},{"index":3,"sense_key":"bid%2:32:02::","synset_id":"bid.v.03"},{"index":4,"sense_key":"bid%2:40:01::","synset_id":"bid.v.04"},{"index":5,"sense_key":"bid%2:40:04::","synset_id":"bid.v.05"},{"index":6,"sense_key":"bid%2:32:00::","synset_id":"invite.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bid%2:40:00::","call_ids":["semeval2013.d003.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bid%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039549999999999996,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.705601957975887}
{"item_id":"semeval2013.d003.s015.t001","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"},{"index":2,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":3,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021009999999999998,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.478217541996855}
{"item_id":"semeval2013.d003.s015.t002","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019649999999999998,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7425211669760756}
{"item_id":"semeval2013.d003.s015.t003","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":2,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":3,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":4,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"},{"index":5,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d003.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004456,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.006561874994077}
{"item_id":"semeval2013.d003.s015.t004","gold_sense_keys":["contract%1:10:00::"],"candidates":[{"index":1,"sense_key":"contract%1:10:00::","synset_id":"contract.n.01"},{"index":2,"sense_key":"contract%1:10:01::","synset_id":"contract.n.02"},{"index":3,"sense_key":"contract%1:04:01::","synset_id":"contract.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contract%1:10:00::","call_ids":["semeval2013.d003.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contract%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046609999999999995,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.9057879579486325}
{"item_id":"semeval2013.d003.s015.t005","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1151,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00049915,"input_uncached_usd":0.000057549999999999996,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.31942783301929}
{"item_id":"semeval2013.d003.s016.t000","gold_sense_keys":["china%1:15:00::"],"candidates":[{"index":1,"sense_key":"china%1:15:00::","synset_id":"china.n.01"},{"index":2,"sense_key":"china%1:06:00::","synset_id":"china.n.02"},{"index":3,"sense_key":"china%1:15:01::","synset_id":"taiwan.n.01"},{"index":4,"sense_key":"china%1:06:01::","synset_id":"chinaware.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"china%1:15:00::","call_ids":["semeval2013.d003.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"china%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051925,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.339635291020386}
{"item_id":"semeval2013.d003.s016.t002","gold_sense_keys":["power%1:14:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"power%1:14:00::","call_ids":["semeval2013.d003.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"power%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022185,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.064780707994942}
{"item_id":"semeval2013.d003.s017.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":690,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006553,"input_uncached_usd":0.0000345,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.04609524999978}
{"item_id":"semeval2013.d003.s017.t001","gold_sense_keys":["par%1:26:00::"],"candidates":[{"index":1,"sense_key":"par%1:23:00::","synset_id":"par.n.01"},{"index":2,"sense_key":"par%1:26:00::","synset_id":"equality.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"par%1:26:00::","call_ids":["semeval2013.d003.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"par%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015624999999999998,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8140622909995727}
{"item_id":"semeval2013.d003.s017.t003","gold_sense_keys":["commitment%1:04:02::"],"candidates":[{"index":1,"sense_key":"commitment%1:07:01::","synset_id":"committedness.n.01"},{"index":2,"sense_key":"commitment%1:04:00::","synset_id":"commitment.n.02"},{"index":3,"sense_key":"commitment%1:04:02::","synset_id":"commitment.n.03"},{"index":4,"sense_key":"commitment%1:10:00::","synset_id":"commitment.n.04"},{"index":5,"sense_key":"commitment%1:04:01::","synset_id":"commitment.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"commitment%1:04:02::","call_ids":["semeval2013.d003.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"commitment%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018764999999999998,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1806293340050615}
{"item_id":"semeval2013.d003.s018.t000","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d003.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027025,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.854779582994524}
{"item_id":"semeval2013.d003.s018.t001","gold_sense_keys":["bid%1:10:00::"],"candidates":[{"index":1,"sense_key":"bid%1:10:03::","synset_id":"command.n.01"},{"index":2,"sense_key":"bid%1:04:00::","synset_id":"bid.n.02"},{"index":3,"sense_key":"bid%1:10:00::","synset_id":"bid.n.03"},{"index":4,"sense_key":"bid%1:10:01::","synset_id":"bid.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bid%1:10:00::","call_ids":["semeval2013.d003.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bid%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010619999999999999,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2353367090108804}
{"item_id":"semeval2013.d003.s018.t002","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"offer%1:10:00::","call_ids":["semeval2013.d003.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031015000000000003,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.558394500054419}
{"item_id":"semeval2013.d003.s018.t003","gold_sense_keys":["barrel%1:23:01::"],"candidates":[{"index":1,"sense_key":"barrel%1:06:01::","synset_id":"barrel.n.01"},{"index":2,"sense_key":"barrel%1:06:00::","synset_id":"barrel.n.02"},{"index":3,"sense_key":"barrel%1:25:00::","synset_id":"barrel.n.03"},{"index":4,"sense_key":"barrel%1:23:04::","synset_id":"barrel.n.04"},{"index":5,"sense_key":"barrel%1:23:01::","synset_id":"barrel.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"barrel%1:23:01::","call_ids":["semeval2013.d003.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"barrel%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026149999999999996,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6623135000118054}
{"item_id":"semeval2013.d003.s018.t004","gold_sense_keys":["fee%1:21:00::"],"candidates":[{"index":1,"sense_key":"fee%1:21:00::","synset_id":"fee.n.01"},{"index":2,"sense_key":"fee%1:21:01::","synset_id":"fee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fee%1:21:00::","call_ids":["semeval2013.d003.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fee%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045999999999999996,"input_uncached_usd":0.0000184,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.346082999953069}
{"item_id":"semeval2013.d003.s018.t005","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":3,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":4,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2013.d003.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001302,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6616921669919975}
{"item_id":"semeval2013.d003.s018.t006","gold_sense_keys":["output%1:04:00::","output%1:06:01::","output%1:23:00::"],"candidates":[{"index":1,"sense_key":"output%1:06:00::","synset_id":"end_product.n.01"},{"index":2,"sense_key":"output%1:04:00::","synset_id":"output.n.02"},{"index":3,"sense_key":"output%1:10:02::","synset_id":"output_signal.n.01"},{"index":4,"sense_key":"output%1:23:00::","synset_id":"output.n.04"},{"index":5,"sense_key":"output%1:06:01::","synset_id":"output.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"output%1:10:02::","call_ids":["semeval2013.d003.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"output%1:10:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007475,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.482446333975531}
{"item_id":"semeval2013.d003.s018.t007","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d003.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032195,"input_uncached_usd":0.00003395,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.096410166006535}
{"item_id":"semeval2013.d003.s018.t008","gold_sense_keys":["field%1:15:05::"],"candidates":[{"index":1,"sense_key":"field%1:15:00::","synset_id":"field.n.01"},{"index":2,"sense_key":"field%1:15:04::","synset_id":"battlefield.n.01"},{"index":3,"sense_key":"field%1:15:01::","synset_id":"field.n.03"},{"index":4,"sense_key":"field%1:09:00::","synset_id":"discipline.n.01"},{"index":5,"sense_key":"field%1:19:00::","synset_id":"field.n.05"},{"index":6,"sense_key":"field%1:04:00::","synset_id":"field.n.06"},{"index":7,"sense_key":"field%1:26:00::","synset_id":"sphere.n.01"},{"index":8,"sense_key":"field%1:15:02::","synset_id":"playing_field.n.02"},{"index":9,"sense_key":"field%1:17:00::","synset_id":"plain.n.01"},{"index":10,"sense_key":"field%1:14:00::","synset_id":"field.n.10"},{"index":11,"sense_key":"field%1:15:03::","synset_id":"field.n.11"},{"index":12,"sense_key":"field%1:14:01::","synset_id":"field.n.12"},{"index":13,"sense_key":"field%1:14:02::","synset_id":"field.n.13"},{"index":14,"sense_key":"field%1:15:05::","synset_id":"field.n.14"},{"index":15,"sense_key":"field%1:14:03::","synset_id":"field.n.15"},{"index":16,"sense_key":"field%1:09:01::","synset_id":"field.n.16"},{"index":17,"sense_key":"field%1:06:00::","synset_id":"airfield.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":14,"chosen_sense_key":"field%1:15:05::","call_ids":["semeval2013.d003.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":14,"predicted_sense_key":"field%1:15:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1091,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00057295,"input_uncached_usd":0.00005455,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.427674583974294}
{"item_id":"semeval2013.d004.s000.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003186,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.066681209020317}
{"item_id":"semeval2013.d004.s000.t001","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":560,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.000572,"input_uncached_usd":0.000028,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.759580375044607}
{"item_id":"semeval2013.d004.s000.t002","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"support%1:04:04::","call_ids":["semeval2013.d004.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"support%1:04:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034764999999999997,"input_uncached_usd":0.00003405,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.648113792005461}
{"item_id":"semeval2013.d004.s001.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037154999999999995,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.544615457998589}
{"item_id":"semeval2013.d004.s001.t002","gold_sense_keys":["state%1:14:00::","state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005225500000000001,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.80485391698312}
{"item_id":"semeval2013.d004.s001.t003","gold_sense_keys":["shackle%1:06:00::"],"candidates":[{"index":1,"sense_key":"shackle%1:06:00::","synset_id":"shackle.n.01"},{"index":2,"sense_key":"shackle%1:06:01::","synset_id":"shackle.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shackle%1:06:00::","call_ids":["semeval2013.d004.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shackle%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":289,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00027685,"input_uncached_usd":0.000014449999999999999,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.654799250012729}
{"item_id":"semeval2013.d004.s002.t001","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032485,"input_uncached_usd":0.00003685,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.060561791004147}
{"item_id":"semeval2013.d004.s002.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":668,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003214,"input_uncached_usd":0.0000334,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9615313750109635}
{"item_id":"semeval2013.d004.s002.t003","gold_sense_keys":["protection%1:21:00::"],"candidates":[{"index":1,"sense_key":"protection%1:04:00::","synset_id":"protection.n.01"},{"index":2,"sense_key":"protection%1:06:00::","synset_id":"protective_covering.n.01"},{"index":3,"sense_key":"protection%1:21:00::","synset_id":"security.n.02"},{"index":4,"sense_key":"protection%1:26:00::","synset_id":"protection.n.04"},{"index":5,"sense_key":"protection%1:04:02::","synset_id":"auspices.n.01"},{"index":6,"sense_key":"protection%1:04:03::","synset_id":"protection.n.06"},{"index":7,"sense_key":"protection%1:04:01::","synset_id":"protection.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"protection%1:21:00::","call_ids":["semeval2013.d004.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"protection%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021375,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5902877500047907}
{"item_id":"semeval2013.d004.s002.t004","gold_sense_keys":["fund%1:21:00::"],"candidates":[{"index":1,"sense_key":"fund%1:21:00::","synset_id":"fund.n.01"},{"index":2,"sense_key":"fund%1:21:01::","synset_id":"store.n.02"},{"index":3,"sense_key":"fund%1:14:00::","synset_id":"investment_company.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fund%1:21:00::","call_ids":["semeval2013.d004.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fund%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":342,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004075,"input_uncached_usd":0.0000171,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.337327209010255}
{"item_id":"semeval2013.d004.s002.t005","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":3,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":4,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d004.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004094,"input_uncached_usd":0.000018999999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.772621874988545}
{"item_id":"semeval2013.d004.s003.t000","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d004.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011249999999999998,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2709272500360385}
{"item_id":"semeval2013.d004.s003.t002","gold_sense_keys":["broadcaster%1:18:00::"],"candidates":[{"index":1,"sense_key":"broadcaster%1:18:00::","synset_id":"broadcaster.n.01"},{"index":2,"sense_key":"broadcaster%1:06:00::","synset_id":"broadcaster.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"broadcaster%1:18:00::","call_ids":["semeval2013.d004.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"broadcaster%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":326,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003811,"input_uncached_usd":0.0000163,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.461963000008836}
{"item_id":"semeval2013.d004.s003.t003","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00096065,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.406073416990694}
{"item_id":"semeval2013.d004.s003.t004","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027545,"input_uncached_usd":0.00003865,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.826848540978972}
{"item_id":"semeval2013.d004.s004.t000","gold_sense_keys":["institution%1:14:00::"],"candidates":[{"index":1,"sense_key":"institution%1:14:00::","synset_id":"institution.n.01"},{"index":2,"sense_key":"institution%1:06:00::","synset_id":"institution.n.02"},{"index":3,"sense_key":"institution%1:09:00::","synset_id":"institution.n.03"},{"index":4,"sense_key":"institution%1:04:00::","synset_id":"initiation.n.02"},{"index":5,"sense_key":"institution%1:06:01::","synset_id":"mental_hospital.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"institution%1:14:00::","call_ids":["semeval2013.d004.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"institution%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002875,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.366088750015479}
{"item_id":"semeval2013.d004.s004.t001","gold_sense_keys":["restriction%1:09:00::"],"candidates":[{"index":1,"sense_key":"restriction%1:09:00::","synset_id":"restriction.n.01"},{"index":2,"sense_key":"restriction%1:04:00::","synset_id":"limitation.n.05"},{"index":3,"sense_key":"restriction%1:04:01::","synset_id":"restriction.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"restriction%1:04:00::","call_ids":["semeval2013.d004.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"restriction%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004105,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.66784850001568}
{"item_id":"semeval2013.d004.s004.t002","gold_sense_keys":["acceptance%1:04:03::"],"candidates":[{"index":1,"sense_key":"acceptance%1:09:00::","synset_id":"credence.n.01"},{"index":2,"sense_key":"acceptance%1:04:00::","synset_id":"adoption.n.01"},{"index":3,"sense_key":"acceptance%1:26:00::","synset_id":"acceptance.n.03"},{"index":4,"sense_key":"acceptance%1:10:01::","synset_id":"acceptance.n.04"},{"index":5,"sense_key":"acceptance%1:21:00::","synset_id":"acceptance.n.05"},{"index":6,"sense_key":"acceptance%1:07:00::","synset_id":"toleration.n.01"},{"index":7,"sense_key":"acceptance%1:04:03::","synset_id":"acceptance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"acceptance%1:04:03::","call_ids":["semeval2013.d004.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"acceptance%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029194999999999997,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.808495874982327}
{"item_id":"semeval2013.d004.s004.t003","gold_sense_keys":["money%1:21:00::"],"candidates":[{"index":1,"sense_key":"money%1:21:00::","synset_id":"money.n.01"},{"index":2,"sense_key":"money%1:21:02::","synset_id":"money.n.02"},{"index":3,"sense_key":"money%1:21:01::","synset_id":"money.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"money%1:21:02::","call_ids":["semeval2013.d004.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"money%1:21:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038449999999999997,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.300541417032946}
{"item_id":"semeval2013.d004.s004.t004","gold_sense_keys":["limit%1:23:00::"],"candidates":[{"index":1,"sense_key":"limit%1:07:00::","synset_id":"limit.n.01"},{"index":2,"sense_key":"limit%1:28:00::","synset_id":"terminus_ad_quem.n.01"},{"index":3,"sense_key":"limit%1:15:02::","synset_id":"limit.n.03"},{"index":4,"sense_key":"limit%1:15:00::","synset_id":"limit.n.04"},{"index":5,"sense_key":"limit%1:23:01::","synset_id":"limit.n.05"},{"index":6,"sense_key":"limit%1:23:00::","synset_id":"limit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"limit%1:23:00::","call_ids":["semeval2013.d004.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"limit%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049265,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.436099541955628}
{"item_id":"semeval2013.d004.s004.t005","gold_sense_keys":["manager%1:18:00::"],"candidates":[{"index":1,"sense_key":"manager%1:18:00::","synset_id":"director.n.01"},{"index":2,"sense_key":"manager%1:18:01::","synset_id":"coach.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"manager%1:18:00::","call_ids":["semeval2013.d004.s004.t005__v1__a1","semeval2013.d004.s004.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"manager%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":731,"cached_input_tokens":0,"output_tokens":1057,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045934999999999997,"input_uncached_usd":0.000036549999999999994,"input_cached_usd":0.0,"output_usd":0.0004228,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.409758166002575}
{"item_id":"semeval2013.d004.s004.t006","gold_sense_keys":["bonus%1:21:00::"],"candidates":[{"index":1,"sense_key":"bonus%1:09:00::","synset_id":"bonus.n.01"},{"index":2,"sense_key":"bonus%1:21:00::","synset_id":"bonus.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bonus%1:21:00::","call_ids":["semeval2013.d004.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bonus%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003825,"input_uncached_usd":0.0000177,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.174732084036805}
{"item_id":"semeval2013.d004.s004.t007","gold_sense_keys":["charge%1:21:02::"],"candidates":[{"index":1,"sense_key":"charge%1:04:01::","synset_id":"charge.n.01"},{"index":2,"sense_key":"charge%1:10:00::","synset_id":"charge.n.02"},{"index":3,"sense_key":"charge%1:21:02::","synset_id":"charge.n.03"},{"index":4,"sense_key":"charge%1:19:00::","synset_id":"charge.n.04"},{"index":5,"sense_key":"charge%1:04:00::","synset_id":"care.n.05"},{"index":6,"sense_key":"charge%1:04:03::","synset_id":"mission.n.03"},{"index":7,"sense_key":"charge%1:18:00::","synset_id":"charge.n.07"},{"index":8,"sense_key":"charge%1:21:01::","synset_id":"charge.n.08"},{"index":9,"sense_key":"charge%1:16:00::","synset_id":"cathexis.n.01"},{"index":10,"sense_key":"charge%1:12:00::","synset_id":"bang.n.04"},{"index":11,"sense_key":"charge%1:10:03::","synset_id":"charge.n.11"},{"index":12,"sense_key":"charge%1:10:01::","synset_id":"commission.n.06"},{"index":13,"sense_key":"charge%1:10:04::","synset_id":"accusation.n.02"},{"index":14,"sense_key":"charge%1:06:01::","synset_id":"charge.n.14"},{"index":15,"sense_key":"charge%1:06:00::","synset_id":"charge.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"charge%1:21:02::","call_ids":["semeval2013.d004.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"charge%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00015624999999999998,"input_uncached_usd":0.000047449999999999995,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.562700041977223}
{"item_id":"semeval2013.d004.s005.t001","gold_sense_keys":["loan%1:21:00::"],"candidates":[{"index":1,"sense_key":"loan%1:21:00::","synset_id":"loan.n.01"},{"index":2,"sense_key":"loan%1:10:00::","synset_id":"loanword.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"loan%1:21:00::","call_ids":["semeval2013.d004.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"loan%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040885,"input_uncached_usd":0.000018449999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.798005332995672}
{"item_id":"semeval2013.d004.s006.t000","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":6,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":7,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rest%1:24:00::","call_ids":["semeval2013.d004.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00008915,"input_uncached_usd":0.00003155,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.7676220419816673}
{"item_id":"semeval2013.d004.s006.t001","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":3,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":4,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"amount%1:03:00::","call_ids":["semeval2013.d004.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"amount%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00079825,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.520927542005666}
{"item_id":"semeval2013.d004.s006.t003","gold_sense_keys":["sale%1:04:00::"],"candidates":[{"index":1,"sense_key":"sale%1:04:00::","synset_id":"sale.n.01"},{"index":2,"sense_key":"sale%1:04:02::","synset_id":"sale.n.02"},{"index":3,"sense_key":"sale%1:04:01::","synset_id":"sale.n.03"},{"index":4,"sense_key":"sale%1:26:00::","synset_id":"sale.n.04"},{"index":5,"sense_key":"sale%1:10:00::","synset_id":"sale.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sale%1:04:00::","call_ids":["semeval2013.d004.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sale%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009063,"input_uncached_usd":0.0000295,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.808040375006385}
{"item_id":"semeval2013.d004.s007.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":770,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022409999999999997,"input_uncached_usd":0.0000385,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.744316208001692}
{"item_id":"semeval2013.d004.s008.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.000249,"input_uncached_usd":0.0000378,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.271028209012002}
{"item_id":"semeval2013.d004.s008.t001","gold_sense_keys":["height%1:26:00::"],"candidates":[{"index":1,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":2,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":3,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":4,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"height%1:26:00::","call_ids":["semeval2013.d004.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"height%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010779999999999999,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.8313542500254698}
{"item_id":"semeval2013.d004.s008.t002","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d004.s008.t002__v1__a1","semeval2013.d004.s008.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038694999999999995,"input_uncached_usd":0.00004135,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.603713290998712}
{"item_id":"semeval2013.d004.s008.t003","gold_sense_keys":["investment%1:04:00::","investment%1:21:00::"],"candidates":[{"index":1,"sense_key":"investment%1:04:00::","synset_id":"investing.n.01"},{"index":2,"sense_key":"investment%1:21:00::","synset_id":"investment.n.02"},{"index":3,"sense_key":"investment%1:07:00::","synset_id":"investment.n.03"},{"index":4,"sense_key":"investment%1:05:00::","synset_id":"investment.n.04"},{"index":5,"sense_key":"investment%1:04:01::","synset_id":"investment.n.05"},{"index":6,"sense_key":"investment%1:04:02::","synset_id":"investment.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"investment%1:04:00::","call_ids":["semeval2013.d004.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"investment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00067375,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.246592834009789}
{"item_id":"semeval2013.d004.s008.t004","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":756,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027459999999999995,"input_uncached_usd":0.0000378,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.58278054202674}
{"item_id":"semeval2013.d004.s009.t000","gold_sense_keys":["cash%1:21:00::"],"candidates":[{"index":1,"sense_key":"cash%1:21:00::","synset_id":"cash.n.01"},{"index":2,"sense_key":"cash%1:21:02::","synset_id":"cash.n.02"},{"index":3,"sense_key":"cash%1:18:00::","synset_id":"cash.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cash%1:21:00::","call_ids":["semeval2013.d004.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cash%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020739999999999997,"input_uncached_usd":0.000021799999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.967151292017661}
{"item_id":"semeval2013.d004.s009.t001","gold_sense_keys":["liability%1:26:00::"],"candidates":[{"index":1,"sense_key":"liability%1:26:01::","synset_id":"liability.n.01"},{"index":2,"sense_key":"liability%1:26:00::","synset_id":"indebtedness.n.01"},{"index":3,"sense_key":"liability%1:07:00::","synset_id":"liability.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"liability%1:26:00::","call_ids":["semeval2013.d004.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"liability%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":80,"reasoning_output_tokens":64},"cost":{"currency":"USD","total_usd":0.00005245,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.000032,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.4854996670037508}
{"item_id":"semeval2013.d004.s009.t002","gold_sense_keys":["state%1:14:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"state%1:14:01::","call_ids":["semeval2013.d004.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"state%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037469999999999995,"input_uncached_usd":0.000035499999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.639252250024583}
{"item_id":"semeval2013.d004.s010.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d004.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022229999999999998,"input_uncached_usd":0.0000367,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.07935474999249}
{"item_id":"semeval2013.d004.s010.t003","gold_sense_keys":["capital%1:21:00::"],"candidates":[{"index":1,"sense_key":"capital%1:21:01::","synset_id":"capital.n.01"},{"index":2,"sense_key":"capital%1:21:00::","synset_id":"capital.n.02"},{"index":3,"sense_key":"capital%1:15:00::","synset_id":"capital.n.03"},{"index":4,"sense_key":"capital%1:10:00::","synset_id":"capital.n.04"},{"index":5,"sense_key":"capital%1:15:01::","synset_id":"capital.n.05"},{"index":6,"sense_key":"capital%1:14:00::","synset_id":"capital.n.06"},{"index":7,"sense_key":"capital%1:10:01::","synset_id":"das_kapital.n.01"},{"index":8,"sense_key":"capital%1:06:00::","synset_id":"capital.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"capital%1:21:00::","call_ids":["semeval2013.d004.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"capital%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029309999999999997,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.235703625017777}
{"item_id":"semeval2013.d004.s010.t004","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":4,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":5,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d004.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003386,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.211918208980933}
{"item_id":"semeval2013.d004.s010.t006","gold_sense_keys":["circle%1:14:00::"],"candidates":[{"index":1,"sense_key":"circle%1:25:00::","synset_id":"circle.n.01"},{"index":2,"sense_key":"circle%1:14:00::","synset_id":"set.n.05"},{"index":3,"sense_key":"circle%1:25:01::","synset_id":"circle.n.03"},{"index":4,"sense_key":"circle%1:04:00::","synset_id":"lap.n.05"},{"index":5,"sense_key":"circle%1:06:01::","synset_id":"traffic_circle.n.01"},{"index":6,"sense_key":"circle%1:06:03::","synset_id":"r-2.n.01"},{"index":7,"sense_key":"circle%1:06:02::","synset_id":"circle.n.07"},{"index":8,"sense_key":"circle%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"circle%1:14:00::","call_ids":["semeval2013.d004.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"circle%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026924999999999996,"input_uncached_usd":0.000032449999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.828786250029225}
{"item_id":"semeval2013.d004.s012.t000","gold_sense_keys":["support%1:21:01::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"support%1:21:01::","call_ids":["semeval2013.d004.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"support%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003271,"input_uncached_usd":0.000039099999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.503482083033305}
{"item_id":"semeval2013.d005.s000.t000","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":382,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002303,"input_uncached_usd":0.0000191,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.413944499974605}
{"item_id":"semeval2013.d005.s001.t002","gold_sense_keys":["row%1:14:00::"],"candidates":[{"index":1,"sense_key":"row%1:14:00::","synset_id":"row.n.01"},{"index":2,"sense_key":"row%1:10:00::","synset_id":"quarrel.n.01"},{"index":3,"sense_key":"row%1:17:00::","synset_id":"row.n.03"},{"index":4,"sense_key":"row%1:06:00::","synset_id":"course.n.08"},{"index":5,"sense_key":"row%1:14:01::","synset_id":"row.n.05"},{"index":6,"sense_key":"row%1:07:00::","synset_id":"row.n.06"},{"index":7,"sense_key":"row%1:04:00::","synset_id":"rowing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"row%1:14:00::","call_ids":["semeval2013.d005.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"row%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018155,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.915606041031424}
{"item_id":"semeval2013.d005.s002.t000","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d005.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031545,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.414023250050377}
{"item_id":"semeval2013.d005.s002.t001","gold_sense_keys":["testimony%1:10:00::"],"candidates":[{"index":1,"sense_key":"testimony%1:10:00::","synset_id":"testimony.n.01"},{"index":2,"sense_key":"testimony%1:10:01::","synset_id":"testimony.n.02"},{"index":3,"sense_key":"testimony%1:10:02::","synset_id":"testimony.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"testimony%1:10:00::","call_ids":["semeval2013.d005.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"testimony%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.000816,"input_uncached_usd":0.000016,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.942608957993798}
{"item_id":"semeval2013.d005.s003.t000","gold_sense_keys":["in_the_end%4:02:00::"],"candidates":[{"index":1,"sense_key":"in_the_end%4:02:02::","synset_id":"in_the_long_run.r.01"},{"index":2,"sense_key":"in_the_end%4:02:00::","synset_id":"ultimately.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"in_the_end%4:02:00::","call_ids":["semeval2013.d005.s003.t000__v1__a1","semeval2013.d005.s003.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"in_the_end%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":2336,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00097465,"input_uncached_usd":0.00004025,"input_cached_usd":0.0,"output_usd":0.0009343999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.273621583008207}
{"item_id":"semeval2013.d005.s003.t001","gold_sense_keys":["defense%1:14:00::"],"candidates":[{"index":1,"sense_key":"defense%1:04:00::","synset_id":"defense.n.01"},{"index":2,"sense_key":"defense%1:04:03::","synset_id":"defense.n.02"},{"index":3,"sense_key":"defense%1:14:01::","synset_id":"defense.n.03"},{"index":4,"sense_key":"defense%1:10:00::","synset_id":"defense.n.04"},{"index":5,"sense_key":"defense%1:22:00::","synset_id":"defense_mechanism.n.01"},{"index":6,"sense_key":"defense%1:14:02::","synset_id":"department_of_defense.n.01"},{"index":7,"sense_key":"defense%1:14:00::","synset_id":"defense.n.07"},{"index":8,"sense_key":"defense%1:10:01::","synset_id":"refutation.n.01"},{"index":9,"sense_key":"defense%1:14:03::","synset_id":"defense.n.09"},{"index":10,"sense_key":"defense%1:06:00::","synset_id":"defensive_structure.n.01"},{"index":11,"sense_key":"defense%1:04:02::","synset_id":"defense.n.11"},{"index":12,"sense_key":"defense%1:04:01::","synset_id":"defense.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"defense%1:14:00::","call_ids":["semeval2013.d005.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"defense%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":870,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002803,"input_uncached_usd":0.0000435,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.298229208972771}
{"item_id":"semeval2013.d005.s003.t002","gold_sense_keys":["message%1:10:01::"],"candidates":[{"index":1,"sense_key":"message%1:10:01::","synset_id":"message.n.01"},{"index":2,"sense_key":"message%1:10:00::","synset_id":"message.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"message%1:10:01::","call_ids":["semeval2013.d005.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"message%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035669999999999994,"input_uncached_usd":0.0000175,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.939805624948349}
{"item_id":"semeval2013.d005.s003.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013739999999999998,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.083581749990117}
{"item_id":"semeval2013.d005.s003.t004","gold_sense_keys":["rape%1:04:00::"],"candidates":[{"index":1,"sense_key":"rape%1:20:00::","synset_id":"rape.n.01"},{"index":2,"sense_key":"rape%1:04:01::","synset_id":"rape.n.02"},{"index":3,"sense_key":"rape%1:04:00::","synset_id":"rape.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rape%1:04:00::","call_ids":["semeval2013.d005.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rape%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":209,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010305,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.0000836,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2308801659964956}
{"item_id":"semeval2013.d005.s003.t005","gold_sense_keys":["child%1:18:00::"],"candidates":[{"index":1,"sense_key":"child%1:18:00::","synset_id":"child.n.01"},{"index":2,"sense_key":"child%1:18:01::","synset_id":"child.n.02"},{"index":3,"sense_key":"child%1:18:06::","synset_id":"child.n.03"},{"index":4,"sense_key":"child%1:18:07::","synset_id":"child.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"child%1:18:00::","call_ids":["semeval2013.d005.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"child%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007457,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.0921178329736}
{"item_id":"semeval2013.d005.s003.t006","gold_sense_keys":["bed%1:06:00::"],"candidates":[{"index":1,"sense_key":"bed%1:06:00::","synset_id":"bed.n.01"},{"index":2,"sense_key":"bed%1:06:01::","synset_id":"bed.n.02"},{"index":3,"sense_key":"bed%1:17:00::","synset_id":"bed.n.03"},{"index":4,"sense_key":"bed%1:17:01::","synset_id":"bed.n.04"},{"index":5,"sense_key":"bed%1:15:00::","synset_id":"seam.n.03"},{"index":6,"sense_key":"bed%1:06:04::","synset_id":"layer.n.01"},{"index":7,"sense_key":"bed%1:06:03::","synset_id":"bed.n.07"},{"index":8,"sense_key":"bed%1:06:02::","synset_id":"bed.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"bed%1:06:00::","call_ids":["semeval2013.d005.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"bed%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013905,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5777792499866337}
{"item_id":"semeval2013.d005.s003.t007","gold_sense_keys":["house%1:06:00::"],"candidates":[{"index":1,"sense_key":"house%1:06:00::","synset_id":"house.n.01"},{"index":2,"sense_key":"house%1:14:01::","synset_id":"firm.n.01"},{"index":3,"sense_key":"house%1:14:04::","synset_id":"house.n.03"},{"index":4,"sense_key":"house%1:14:03::","synset_id":"house.n.04"},{"index":5,"sense_key":"house%1:14:02::","synset_id":"house.n.05"},{"index":6,"sense_key":"house%1:14:00::","synset_id":"house.n.06"},{"index":7,"sense_key":"house%1:04:00::","synset_id":"house.n.07"},{"index":8,"sense_key":"house%1:15:00::","synset_id":"sign_of_the_zodiac.n.01"},{"index":9,"sense_key":"house%1:14:06::","synset_id":"house.n.09"},{"index":10,"sense_key":"house%1:14:05::","synset_id":"family.n.01"},{"index":11,"sense_key":"house%1:06:02::","synset_id":"theater.n.01"},{"index":12,"sense_key":"house%1:06:01::","synset_id":"house.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"house%1:06:00::","call_ids":["semeval2013.d005.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"house%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":750,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005559,"input_uncached_usd":0.0000375,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.271809957979713}
{"item_id":"semeval2013.d005.s004.t001","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021549999999999998,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5722517909598537}
{"item_id":"semeval2013.d005.s004.t002","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["semeval2013.d005.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008276500000000001,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.404933042009361}
{"item_id":"semeval2013.d005.s004.t003","gold_sense_keys":["family%1:14:00::","family%1:14:02::"],"candidates":[{"index":1,"sense_key":"family%1:14:02::","synset_id":"family.n.01"},{"index":2,"sense_key":"family%1:14:00::","synset_id":"family.n.02"},{"index":3,"sense_key":"family%1:14:04::","synset_id":"class.n.01"},{"index":4,"sense_key":"family%1:14:01::","synset_id":"family.n.04"},{"index":5,"sense_key":"family%1:18:00::","synset_id":"kin.n.01"},{"index":6,"sense_key":"family%1:14:03::","synset_id":"family.n.06"},{"index":7,"sense_key":"family%1:14:06::","synset_id":"syndicate.n.01"},{"index":8,"sense_key":"family%1:14:05::","synset_id":"family.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"family%1:14:00::","call_ids":["semeval2013.d005.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"family%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032145000000000003,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.332358833984472}
{"item_id":"semeval2013.d005.s004.t004","gold_sense_keys":["end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d005.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":896,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006656,"input_uncached_usd":0.0000448,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.111630791972857}
{"item_id":"semeval2013.d005.s004.t005","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005880500000000001,"input_uncached_usd":0.000018449999999999998,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.770321292045992}
{"item_id":"semeval2013.d005.s004.t006","gold_sense_keys":["imposition%1:04:02::"],"candidates":[{"index":1,"sense_key":"imposition%1:04:02::","synset_id":"imposition.n.01"},{"index":2,"sense_key":"imposition%1:09:00::","synset_id":"imposition.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"imposition%1:04:02::","call_ids":["semeval2013.d005.s004.t006__v1__a1","semeval2013.d005.s004.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"imposition%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00061575,"input_uncached_usd":0.000039750000000000004,"input_cached_usd":0.0,"output_usd":0.0005759999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.066918958036695}
{"item_id":"semeval2013.d005.s004.t007","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002038,"input_uncached_usd":0.0000182,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.522549249988515}
{"item_id":"semeval2013.d005.s005.t002","gold_sense_keys":["filing%1:10:00::"],"candidates":[{"index":1,"sense_key":"filing%1:10:00::","synset_id":"filing.n.01"},{"index":2,"sense_key":"filing%1:17:00::","synset_id":"filing.n.02"},{"index":3,"sense_key":"filing%1:04:01::","synset_id":"filing.n.03"},{"index":4,"sense_key":"filing%1:04:00::","synset_id":"filing.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"filing%1:10:00::","call_ids":["semeval2013.d005.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"filing%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028704999999999996,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.580265666998457}
{"item_id":"semeval2013.d005.s005.t003","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00008585,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.8052707090391777}
{"item_id":"semeval2013.d005.s005.t004","gold_sense_keys":["argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"argument%1:09:01::","call_ids":["semeval2013.d005.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"argument%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039965,"input_uncached_usd":0.00003485,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.302594750013668}
{"item_id":"semeval2013.d005.s005.t005","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d005.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006089000000000001,"input_uncached_usd":0.0000393,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.756653582968283}
{"item_id":"semeval2013.d005.s005.t006","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d005.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034094999999999997,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.416192625008989}
{"item_id":"semeval2013.d005.s006.t001","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trial%1:04:03::","call_ids":["semeval2013.d005.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trial%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005002499999999999,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.341481041978113}
{"item_id":"semeval2013.d005.s006.t003","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2013.d005.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027289999999999997,"input_uncached_usd":0.000036099999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.313413458003197}
{"item_id":"semeval2013.d005.s006.t004","gold_sense_keys":["circus%1:04:01::"],"candidates":[{"index":1,"sense_key":"circus%1:14:00::","synset_id":"circus.n.01"},{"index":2,"sense_key":"circus%1:04:00::","synset_id":"circus.n.02"},{"index":3,"sense_key":"circus%1:04:01::","synset_id":"circus.n.03"},{"index":4,"sense_key":"circus%1:06:01::","synset_id":"circus.n.04"},{"index":5,"sense_key":"circus%1:06:00::","synset_id":"circus.n.05"},{"index":6,"sense_key":"circus%1:05:00::","synset_id":"circus.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"circus%1:04:01::","call_ids":["semeval2013.d005.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"circus%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011444999999999999,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2642279170104302}
{"item_id":"semeval2013.d005.s006.t005","gold_sense_keys":["atmosphere%1:26:01::"],"candidates":[{"index":1,"sense_key":"atmosphere%1:26:01::","synset_id":"atmosphere.n.01"},{"index":2,"sense_key":"atmosphere%1:23:00::","synset_id":"standard_atmosphere.n.01"},{"index":3,"sense_key":"atmosphere%1:15:00::","synset_id":"atmosphere.n.03"},{"index":4,"sense_key":"atmosphere%1:26:00::","synset_id":"atmosphere.n.04"},{"index":5,"sense_key":"atmosphere%1:17:00::","synset_id":"atmosphere.n.05"},{"index":6,"sense_key":"atmosphere%1:07:00::","synset_id":"air.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"atmosphere%1:26:01::","call_ids":["semeval2013.d005.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"atmosphere%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021705,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8960573339718394}
{"item_id":"semeval2013.d005.s006.t006","gold_sense_keys":["detail%1:09:00::"],"candidates":[{"index":1,"sense_key":"detail%1:09:00::","synset_id":"detail.n.01"},{"index":2,"sense_key":"detail%1:24:00::","synset_id":"detail.n.02"},{"index":3,"sense_key":"detail%1:10:00::","synset_id":"detail.n.03"},{"index":4,"sense_key":"detail%1:14:01::","synset_id":"detail.n.04"},{"index":5,"sense_key":"detail%1:14:00::","synset_id":"contingent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"detail%1:10:00::","call_ids":["semeval2013.d005.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"detail%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001623,"input_uncached_usd":0.000027899999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8132243750151247}
{"item_id":"semeval2013.d005.s006.t007","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033499999999999996,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.78674183296971}
{"item_id":"semeval2013.d005.s006.t008","gold_sense_keys":["passion%1:12:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"passion%1:12:00::","call_ids":["semeval2013.d005.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"passion%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":599,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036914999999999995,"input_uncached_usd":0.00002995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.445627875044011}
{"item_id":"semeval2013.d005.s007.t000","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009342,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.348273625015281}
{"item_id":"semeval2013.d005.s007.t001","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":612,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024179999999999997,"input_uncached_usd":0.0000306,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.325517082994338}
{"item_id":"semeval2013.d005.s007.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s007.t002__v1__a1","semeval2013.d005.s007.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":981,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005226499999999999,"input_uncached_usd":0.00004905,"input_cached_usd":0.0,"output_usd":0.00047359999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.199334417004138}
{"item_id":"semeval2013.d005.s007.t003","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d005.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":693,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00073225,"input_uncached_usd":0.000034649999999999995,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.030971625004895}
{"item_id":"semeval2013.d005.s007.t005","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":2576,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.00107785,"input_uncached_usd":0.000047449999999999995,"input_cached_usd":0.0,"output_usd":0.0010304,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.08887766697444}
{"item_id":"semeval2013.d005.s008.t000","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trial%1:04:03::","call_ids":["semeval2013.d005.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trial%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055475,"input_uncached_usd":0.000036349999999999996,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.024361875024624}
{"item_id":"semeval2013.d005.s008.t001","gold_sense_keys":["danger%1:04:00::","danger%1:26:00::"],"candidates":[{"index":1,"sense_key":"danger%1:26:00::","synset_id":"danger.n.01"},{"index":2,"sense_key":"danger%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"danger%1:26:01::","synset_id":"danger.n.03"},{"index":4,"sense_key":"danger%1:15:00::","synset_id":"danger.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"danger%1:04:00::","call_ids":["semeval2013.d005.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"danger%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029194999999999997,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.150208292005118}
{"item_id":"semeval2013.d005.s008.t002","gold_sense_keys":["inclusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"inclusion%1:26:00::","synset_id":"inclusion.n.01"},{"index":2,"sense_key":"inclusion%1:24:00::","synset_id":"inclusion.n.02"},{"index":3,"sense_key":"inclusion%1:17:00::","synset_id":"inclusion_body.n.01"},{"index":4,"sense_key":"inclusion%1:04:00::","synset_id":"inclusion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"inclusion%1:04:00::","call_ids":["semeval2013.d005.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"inclusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003165,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.660326834011357}
{"item_id":"semeval2013.d005.s008.t004","gold_sense_keys":["individual%1:03:00::"],"candidates":[{"index":1,"sense_key":"individual%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"individual%1:18:00::","synset_id":"individual.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"individual%1:03:00::","call_ids":["semeval2013.d005.s008.t004__v1__a1","semeval2013.d005.s008.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"individual%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":1313,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00057705,"input_uncached_usd":0.00005185,"input_cached_usd":0.0,"output_usd":0.0005252,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.954495958983898}
{"item_id":"semeval2013.d005.s008.t005","gold_sense_keys":["right%1:07:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"right%1:07:00::","call_ids":["semeval2013.d005.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"right%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002523,"input_uncached_usd":0.000041099999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.182946750021074}
{"item_id":"semeval2013.d005.s009.t003","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"claim%1:04:00::","call_ids":["semeval2013.d005.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"claim%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00075535,"input_uncached_usd":0.000032149999999999995,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.800013167026918}
{"item_id":"semeval2013.d005.s009.t005","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d005.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005697,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.341695042036008}
{"item_id":"semeval2013.d005.s009.t007","gold_sense_keys":["evidence%1:09:00::"],"candidates":[{"index":1,"sense_key":"evidence%1:09:00::","synset_id":"evidence.n.01"},{"index":2,"sense_key":"evidence%1:10:01::","synset_id":"evidence.n.02"},{"index":3,"sense_key":"evidence%1:10:00::","synset_id":"evidence.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"evidence%1:10:00::","call_ids":["semeval2013.d005.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"evidence%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.000084,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.83088545902865}
{"item_id":"semeval2013.d005.s009.t008","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046504999999999995,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.439517666003667}
{"item_id":"semeval2013.d005.s009.t009","gold_sense_keys":["passion%1:12:00::"],"candidates":[{"index":1,"sense_key":"passion%1:12:00::","synset_id":"passion.n.01"},{"index":2,"sense_key":"passion%1:07:00::","synset_id":"heat.n.04"},{"index":3,"sense_key":"passion%1:26:01::","synset_id":"rage.n.03"},{"index":4,"sense_key":"passion%1:16:00::","synset_id":"mania.n.01"},{"index":5,"sense_key":"passion%1:12:02::","synset_id":"passion.n.05"},{"index":6,"sense_key":"passion%1:09:00::","synset_id":"love.n.02"},{"index":7,"sense_key":"passion%1:26:02::","synset_id":"passion.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"passion%1:12:00::","call_ids":["semeval2013.d005.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"passion%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":640,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002688,"input_uncached_usd":0.000032,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.360478583024815}
{"item_id":"semeval2013.d005.s010.t000","gold_sense_keys":["last_word%1:10:00::"],"candidates":[{"index":1,"sense_key":"last_word%1:10:00::","synset_id":"last_word.n.01"},{"index":2,"sense_key":"last_word%1:10:01::","synset_id":"last_word.n.02"},{"index":3,"sense_key":"last_word%1:07:00::","synset_id":"chic.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"last_word%1:10:01::","call_ids":["semeval2013.d005.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"last_word%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056855,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.478390541975386}
{"item_id":"semeval2013.d005.s010.t001","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:00::","call_ids":["semeval2013.d005.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007178499999999999,"input_uncached_usd":0.000045849999999999996,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.576226499979384}
{"item_id":"semeval2013.d005.s011.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d005.s011.t000__v1__a1","semeval2013.d005.s011.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00055105,"input_uncached_usd":0.00005185,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.18625000002794}
{"item_id":"semeval2013.d005.s011.t002","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018649999999999998,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.161314124998171}
{"item_id":"semeval2013.d005.s011.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1277,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00022385,"input_uncached_usd":0.00006385,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.284427209000569}
{"item_id":"semeval2013.d005.s011.t004","gold_sense_keys":["question%1:07:00::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d005.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040025,"input_uncached_usd":0.00003545,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.524504541943315}
{"item_id":"semeval2013.d005.s011.t005","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022114999999999998,"input_uncached_usd":0.00003555,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8619929169653915}
{"item_id":"semeval2013.d005.s011.t006","gold_sense_keys":["fairness%1:07:00::"],"candidates":[{"index":1,"sense_key":"fairness%1:07:00::","synset_id":"fairness.n.01"},{"index":2,"sense_key":"fairness%1:09:00::","synset_id":"fairness.n.02"},{"index":3,"sense_key":"fairness%1:07:01::","synset_id":"paleness.n.02"},{"index":4,"sense_key":"fairness%1:07:02::","synset_id":"comeliness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fairness%1:09:00::","call_ids":["semeval2013.d005.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fairness%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029085,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.366708832967561}
{"item_id":"semeval2013.d005.s012.t002","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006451,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.428327417001128}
{"item_id":"semeval2013.d005.s012.t003","gold_sense_keys":["cleveland%1:15:00::"],"candidates":[{"index":1,"sense_key":"cleveland%1:15:00::","synset_id":"cleveland.n.01"},{"index":2,"sense_key":"cleveland%1:18:00::","synset_id":"cleveland.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cleveland%1:15:00::","call_ids":["semeval2013.d005.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cleveland%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043995,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.855129042000044}
{"item_id":"semeval2013.d005.s012.t007","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001647,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.885786917002406}
{"item_id":"semeval2013.d005.s012.t008","gold_sense_keys":["carnival%1:04:02::"],"candidates":[{"index":1,"sense_key":"carnival%1:04:00::","synset_id":"carnival.n.01"},{"index":2,"sense_key":"carnival%1:04:02::","synset_id":"circus.n.03"},{"index":3,"sense_key":"carnival%1:04:01::","synset_id":"carnival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"carnival%1:04:02::","call_ids":["semeval2013.d005.s012.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"carnival%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023725,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.158065458002966}
{"item_id":"semeval2013.d005.s012.t009","gold_sense_keys":["atmosphere%1:26:01::"],"candidates":[{"index":1,"sense_key":"atmosphere%1:26:01::","synset_id":"atmosphere.n.01"},{"index":2,"sense_key":"atmosphere%1:23:00::","synset_id":"standard_atmosphere.n.01"},{"index":3,"sense_key":"atmosphere%1:15:00::","synset_id":"atmosphere.n.03"},{"index":4,"sense_key":"atmosphere%1:26:00::","synset_id":"atmosphere.n.04"},{"index":5,"sense_key":"atmosphere%1:17:00::","synset_id":"atmosphere.n.05"},{"index":6,"sense_key":"atmosphere%1:07:00::","synset_id":"air.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"atmosphere%1:26:01::","call_ids":["semeval2013.d005.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"atmosphere%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052545,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.440274374966975}
{"item_id":"semeval2013.d005.s012.t010","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032215,"input_uncached_usd":0.000034149999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.834969584015198}
{"item_id":"semeval2013.d005.s013.t000","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d005.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004413,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.665053166972939}
{"item_id":"semeval2013.d005.s013.t001","gold_sense_keys":["inspiration%1:09:02::"],"candidates":[{"index":1,"sense_key":"inspiration%1:09:02::","synset_id":"inspiration.n.01"},{"index":2,"sense_key":"inspiration%1:06:00::","synset_id":"inspiration.n.02"},{"index":3,"sense_key":"inspiration%1:09:01::","synset_id":"inspiration.n.03"},{"index":4,"sense_key":"inspiration%1:09:00::","synset_id":"divine_guidance.n.01"},{"index":5,"sense_key":"inspiration%1:04:01::","synset_id":"inspiration.n.05"},{"index":6,"sense_key":"inspiration%1:04:00::","synset_id":"inhalation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"inspiration%1:09:02::","call_ids":["semeval2013.d005.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"inspiration%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006793,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.907059624965768}
{"item_id":"semeval2013.d005.s014.t002","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conviction%1:09:00::","call_ids":["semeval2013.d005.s014.t002__v1__a1","semeval2013.d005.s014.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conviction%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1089,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00045125,"input_uncached_usd":0.000054449999999999995,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.968616875004955}
{"item_id":"semeval2013.d005.s014.t003","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"},{"index":2,"sense_key":"half%1:28:00::","synset_id":"half.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"half%1:23:00::","call_ids":["semeval2013.d005.s014.t003__v1__a1","semeval2013.d005.s014.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1051,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00075655,"input_uncached_usd":0.000052550000000000003,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.867702416959219}
{"item_id":"semeval2013.d005.s014.t004","gold_sense_keys":["century%1:28:00::"],"candidates":[{"index":1,"sense_key":"century%1:28:00::","synset_id":"century.n.01"},{"index":2,"sense_key":"century%1:23:00::","synset_id":"hundred.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"century%1:28:00::","call_ids":["semeval2013.d005.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"century%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036329999999999994,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.96645362500567}
{"item_id":"semeval2013.d005.s014.t005","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"argument%1:10:02::","call_ids":["semeval2013.d005.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"argument%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047999999999999996,"input_uncached_usd":0.0000384,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.186456249968614}
{"item_id":"semeval2013.d005.s014.t006","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rule%1:09:00::","call_ids":["semeval2013.d005.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rule%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":974,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005415000000000001,"input_uncached_usd":0.0000487,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.44929579203017}
{"item_id":"semeval2013.d005.s014.t007","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d005.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003583,"input_uncached_usd":0.000044699999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.4095718750031665}
{"item_id":"semeval2013.d005.s014.t010","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s014.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1284,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000301,"input_uncached_usd":0.0000642,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.671409124974161}
{"item_id":"semeval2013.d005.s015.t000","gold_sense_keys":["reason%1:09:01::","reason%1:10:00::","reason%1:10:01::","reason%1:16:00::"],"candidates":[{"index":1,"sense_key":"reason%1:16:00::","synset_id":"reason.n.01"},{"index":2,"sense_key":"reason%1:10:01::","synset_id":"reason.n.02"},{"index":3,"sense_key":"reason%1:09:00::","synset_id":"reason.n.03"},{"index":4,"sense_key":"reason%1:26:00::","synset_id":"rationality.n.01"},{"index":5,"sense_key":"reason%1:10:00::","synset_id":"cause.n.02"},{"index":6,"sense_key":"reason%1:09:01::","synset_id":"reason.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"reason%1:10:00::","call_ids":["semeval2013.d005.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"reason%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032215,"input_uncached_usd":0.000034149999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.275221957999747}
{"item_id":"semeval2013.d005.s015.t001","gold_sense_keys":["concern%1:12:01::"],"candidates":[{"index":1,"sense_key":"concern%1:09:00::","synset_id":"concern.n.01"},{"index":2,"sense_key":"concern%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"concern%1:12:00::","synset_id":"concern.n.03"},{"index":4,"sense_key":"concern%1:09:01::","synset_id":"concern.n.04"},{"index":5,"sense_key":"concern%1:14:00::","synset_id":"business.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"concern%1:09:01::","call_ids":["semeval2013.d005.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"concern%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003199,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.643784624990076}
{"item_id":"semeval2013.d005.s015.t002","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d005.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":2576,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010585,"input_uncached_usd":0.0000281,"input_cached_usd":0.0,"output_usd":0.0010304,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.156845499994233}
{"item_id":"semeval2013.d005.s015.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trial%1:04:03::","call_ids":["semeval2013.d005.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trial%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005789,"input_uncached_usd":0.0000349,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.366397292003967}
{"item_id":"semeval2013.d005.s015.t005","gold_sense_keys":["connecticut%1:15:00::"],"candidates":[{"index":1,"sense_key":"connecticut%1:15:00::","synset_id":"connecticut.n.01"},{"index":2,"sense_key":"connecticut%1:17:00::","synset_id":"connecticut.n.02"},{"index":3,"sense_key":"connecticut%1:15:01::","synset_id":"connecticut.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"connecticut%1:15:00::","call_ids":["semeval2013.d005.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"connecticut%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013529999999999998,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.573619582981337}
{"item_id":"semeval2013.d005.s015.t007","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d005.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024754999999999997,"input_uncached_usd":0.000036349999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.901984750002157}
{"item_id":"semeval2013.d005.s016.t001","gold_sense_keys":["appeal%1:04:00::"],"candidates":[{"index":1,"sense_key":"appeal%1:10:00::","synset_id":"entreaty.n.01"},{"index":2,"sense_key":"appeal%1:07:00::","synset_id":"appeal.n.02"},{"index":3,"sense_key":"appeal%1:04:00::","synset_id":"appeal.n.03"},{"index":4,"sense_key":"appeal%1:10:02::","synset_id":"solicitation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"appeal%1:04:00::","call_ids":["semeval2013.d005.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"appeal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011465,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3676594580174424}
{"item_id":"semeval2013.d005.s016.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044019999999999997,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.914962666982319}
{"item_id":"semeval2013.d005.s016.t003","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"idea%1:09:03::","call_ids":["semeval2013.d005.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"idea%1:09:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":660,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014179999999999998,"input_uncached_usd":0.000032999999999999996,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6652546250261366}
{"item_id":"semeval2013.d005.s016.t004","gold_sense_keys":["communication%1:10:01::"],"candidates":[{"index":1,"sense_key":"communication%1:10:01::","synset_id":"communication.n.01"},{"index":2,"sense_key":"communication%1:03:00::","synset_id":"communication.n.02"},{"index":3,"sense_key":"communication%1:24:00::","synset_id":"communication.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"communication%1:10:01::","call_ids":["semeval2013.d005.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"communication%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041735,"input_uncached_usd":0.000026949999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.222041625005659}
{"item_id":"semeval2013.d005.s016.t006","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003748499999999999,"input_uncached_usd":0.00003565,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.500597208039835}
{"item_id":"semeval2013.d005.s017.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d005.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":983,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036274999999999995,"input_uncached_usd":0.000049149999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.987948458001483}
{"item_id":"semeval2013.d005.s017.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d005.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037094999999999994,"input_uncached_usd":0.00003175,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.42405625001993}
{"item_id":"semeval2013.d005.s017.t003","gold_sense_keys":["execution%1:04:00::"],"candidates":[{"index":1,"sense_key":"execution%1:04:00::","synset_id":"execution.n.01"},{"index":2,"sense_key":"execution%1:04:02::","synset_id":"performance.n.03"},{"index":3,"sense_key":"execution%1:22:00::","synset_id":"execution.n.03"},{"index":4,"sense_key":"execution%1:10:01::","synset_id":"execution.n.04"},{"index":5,"sense_key":"execution%1:10:00::","synset_id":"execution.n.05"},{"index":6,"sense_key":"execution%1:04:03::","synset_id":"execution.n.06"},{"index":7,"sense_key":"execution%1:04:01::","synset_id":"murder.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"execution%1:04:00::","call_ids":["semeval2013.d005.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"execution%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027489999999999996,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.520110583980568}
{"item_id":"semeval2013.d005.s018.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006962499999999999,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.723205415997654}
{"item_id":"semeval2013.d005.s018.t004","gold_sense_keys":["reaction%1:04:00::","reaction%1:09:00::","reaction%1:19:01::"],"candidates":[{"index":1,"sense_key":"reaction%1:22:00::","synset_id":"chemical_reaction.n.01"},{"index":2,"sense_key":"reaction%1:09:00::","synset_id":"reaction.n.02"},{"index":3,"sense_key":"reaction%1:04:00::","synset_id":"reaction.n.03"},{"index":4,"sense_key":"reaction%1:19:00::","synset_id":"reaction.n.04"},{"index":5,"sense_key":"reaction%1:19:01::","synset_id":"reaction.n.05"},{"index":6,"sense_key":"reaction%1:09:01::","synset_id":"reaction.n.06"},{"index":7,"sense_key":"reaction%1:04:01::","synset_id":"reaction.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"reaction%1:04:00::","call_ids":["semeval2013.d005.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"reaction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005556999999999999,"input_uncached_usd":0.000036899999999999996,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.70615258300677}
{"item_id":"semeval2013.d005.s019.t002","gold_sense_keys":["victim%1:18:00::"],"candidates":[{"index":1,"sense_key":"victim%1:18:00::","synset_id":"victim.n.01"},{"index":2,"sense_key":"victim%1:18:01::","synset_id":"victim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"victim%1:18:00::","call_ids":["semeval2013.d005.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"victim%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002877,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.462309875001665}
{"item_id":"semeval2013.d005.s020.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d005.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005424999999999999,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.416771791002247}
{"item_id":"semeval2013.d005.s020.t005","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d005.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002473,"input_uncached_usd":0.000036099999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9511117920046672}
{"item_id":"semeval2013.d005.s020.t007","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024249999999999999,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3490693750209175}
{"item_id":"semeval2013.d005.s020.t008","gold_sense_keys":["jury%1:14:00::"],"candidates":[{"index":1,"sense_key":"jury%1:14:00::","synset_id":"jury.n.01"},{"index":2,"sense_key":"jury%1:14:01::","synset_id":"jury.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"jury%1:14:00::","call_ids":["semeval2013.d005.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"jury%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023469999999999999,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.507018958975095}
{"item_id":"semeval2013.d005.s022.t000","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"argument%1:10:00::","call_ids":["semeval2013.d005.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"argument%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003244,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.7266524999868125}
{"item_id":"semeval2013.d005.s022.t001","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d005.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":644,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024339999999999998,"input_uncached_usd":0.0000322,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.55318254197482}
{"item_id":"semeval2013.d005.s022.t002","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001593,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5096469579730183}
{"item_id":"semeval2013.d005.s022.t003","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trial%1:04:03::","call_ids":["semeval2013.d005.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trial%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057795,"input_uncached_usd":0.00003395,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.734216709039174}
{"item_id":"semeval2013.d005.s022.t004","gold_sense_keys":["interest%1:09:00::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"interest%1:07:02::","call_ids":["semeval2013.d005.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"interest%1:07:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024675,"input_uncached_usd":0.00003555,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.597892249992583}
{"item_id":"semeval2013.d005.s023.t000","gold_sense_keys":["challenge%1:10:01::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"challenge%1:10:01::","call_ids":["semeval2013.d005.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"challenge%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033995,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.726511917018797}
{"item_id":"semeval2013.d005.s023.t002","gold_sense_keys":["month%1:28:01::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"month%1:28:01::","call_ids":["semeval2013.d005.s023.t002__v1__a1","semeval2013.d005.s023.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"month%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":943,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006999500000000001,"input_uncached_usd":0.000047149999999999994,"input_cached_usd":0.0,"output_usd":0.0006527999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.63425924902549}
{"item_id":"semeval2013.d005.s023.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1234,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00022169999999999997,"input_uncached_usd":0.0000617,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.878226333006751}
{"item_id":"semeval2013.d005.s023.t004","gold_sense_keys":["news%1:10:01::","news%1:10:02::"],"candidates":[{"index":1,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":2,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":3,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":4,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":5,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d005.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"news%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":593,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047124999999999993,"input_uncached_usd":0.00002965,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0337864169850945}
{"item_id":"semeval2013.d005.s023.t005","gold_sense_keys":["connecticut%1:15:00::"],"candidates":[{"index":1,"sense_key":"connecticut%1:15:00::","synset_id":"connecticut.n.01"},{"index":2,"sense_key":"connecticut%1:17:00::","synset_id":"connecticut.n.02"},{"index":3,"sense_key":"connecticut%1:15:01::","synset_id":"connecticut.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"connecticut%1:15:00::","call_ids":["semeval2013.d005.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"connecticut%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006458,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.755488499999046}
{"item_id":"semeval2013.d005.s023.t006","gold_sense_keys":["election%1:04:01::"],"candidates":[{"index":1,"sense_key":"election%1:04:01::","synset_id":"election.n.01"},{"index":2,"sense_key":"election%1:04:00::","synset_id":"election.n.02"},{"index":3,"sense_key":"election%1:26:00::","synset_id":"election.n.03"},{"index":4,"sense_key":"election%1:09:00::","synset_id":"election.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"election%1:04:01::","call_ids":["semeval2013.d005.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"election%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044195,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.217266125022434}
{"item_id":"semeval2013.d005.s023.t007","gold_sense_keys":["campaign%1:11:00::"],"candidates":[{"index":1,"sense_key":"campaign%1:11:00::","synset_id":"political_campaign.n.01"},{"index":2,"sense_key":"campaign%1:04:02::","synset_id":"campaign.n.02"},{"index":3,"sense_key":"campaign%1:04:00::","synset_id":"campaign.n.03"},{"index":4,"sense_key":"campaign%1:04:01::","synset_id":"campaign.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"campaign%1:11:00::","call_ids":["semeval2013.d005.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"campaign%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021339999999999998,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4075084580108523}
{"item_id":"semeval2013.d005.s023.t008","gold_sense_keys":["candidate%1:18:01::"],"candidates":[{"index":1,"sense_key":"candidate%1:18:01::","synset_id":"campaigner.n.01"},{"index":2,"sense_key":"candidate%1:18:00::","synset_id":"candidate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"candidate%1:18:01::","call_ids":["semeval2013.d005.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"candidate%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006941,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.237093959003687}
{"item_id":"semeval2013.d005.s024.t002","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1221,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00016984999999999998,"input_uncached_usd":0.00006104999999999999,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.160179666010663}
{"item_id":"semeval2013.d005.s025.t003","gold_sense_keys":["courthouse%1:06:00::"],"candidates":[{"index":1,"sense_key":"courthouse%1:06:01::","synset_id":"courthouse.n.01"},{"index":2,"sense_key":"courthouse%1:06:00::","synset_id":"courthouse.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"courthouse%1:06:00::","call_ids":["semeval2013.d005.s025.t003__v1__a1","semeval2013.d005.s025.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"courthouse%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038564999999999997,"input_uncached_usd":0.00004005,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.7168478340609}
{"item_id":"semeval2013.d005.s026.t000","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":2,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"},{"index":3,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":4,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":5,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":6,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":7,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2013.d005.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031775,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.734717832994647}
{"item_id":"semeval2013.d005.s026.t001","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010439999999999999,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3114599169930443}
{"item_id":"semeval2013.d005.s027.t000","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"},{"index":2,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2013.d005.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006419,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.780165208037943}
{"item_id":"semeval2013.d005.s027.t001","gold_sense_keys":["journalist%1:18:00::"],"candidates":[{"index":1,"sense_key":"journalist%1:18:00::","synset_id":"journalist.n.01"},{"index":2,"sense_key":"journalist%1:18:01::","synset_id":"diarist.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"journalist%1:18:00::","call_ids":["semeval2013.d005.s027.t001__v1__a1","semeval2013.d005.s027.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"journalist%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":672,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030915,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.0002688,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.904924291011412}
{"item_id":"semeval2013.d005.s027.t002","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"claim%1:04:00::","call_ids":["semeval2013.d005.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"claim%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057195,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.794071625045035}
{"item_id":"semeval2013.d005.s027.t003","gold_sense_keys":["news%1:10:00::","news%1:10:01::","news%1:10:03::"],"candidates":[{"index":1,"sense_key":"news%1:10:00::","synset_id":"news.n.01"},{"index":2,"sense_key":"news%1:10:01::","synset_id":"news.n.02"},{"index":3,"sense_key":"news%1:10:02::","synset_id":"news_program.n.01"},{"index":4,"sense_key":"news%1:10:03::","synset_id":"news.n.04"},{"index":5,"sense_key":"news%1:07:00::","synset_id":"newsworthiness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"news%1:10:00::","call_ids":["semeval2013.d005.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"news%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034074999999999996,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9406450419919565}
{"item_id":"semeval2013.d005.s027.t004","gold_sense_keys":["report%1:10:00::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d005.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016675,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.788793083978817}
{"item_id":"semeval2013.d005.s027.t005","gold_sense_keys":["view%1:09:02::","view%1:09:04::","view%1:10:00::"],"candidates":[{"index":1,"sense_key":"view%1:09:02::","synset_id":"position.n.03"},{"index":2,"sense_key":"view%1:09:01::","synset_id":"view.n.02"},{"index":3,"sense_key":"view%1:04:00::","synset_id":"view.n.03"},{"index":4,"sense_key":"view%1:15:00::","synset_id":"view.n.04"},{"index":5,"sense_key":"view%1:09:04::","synset_id":"opinion.n.01"},{"index":6,"sense_key":"view%1:10:00::","synset_id":"opinion.n.02"},{"index":7,"sense_key":"view%1:09:00::","synset_id":"view.n.07"},{"index":8,"sense_key":"view%1:06:00::","synset_id":"scene.n.08"},{"index":9,"sense_key":"view%1:07:00::","synset_id":"horizon.n.02"},{"index":10,"sense_key":"view%1:07:01::","synset_id":"view.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"view%1:09:04::","call_ids":["semeval2013.d005.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"view%1:09:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":789,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027624999999999997,"input_uncached_usd":0.00003945,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.481382708996534}
{"item_id":"semeval2013.d005.s027.t006","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d005.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007353500000000001,"input_uncached_usd":0.000037749999999999996,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.524041500000749}
{"item_id":"semeval2013.d005.s027.t007","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d005.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003106,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.243535125046037}
{"item_id":"semeval2013.d005.s027.t008","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015624999999999998,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4992328329826705}
{"item_id":"semeval2013.d005.s027.t009","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006517000000000001,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.041193332988769}
{"item_id":"semeval2013.d005.s028.t002","gold_sense_keys":["argument%1:09:01::","argument%1:10:02::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"argument%1:09:01::","call_ids":["semeval2013.d005.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"argument%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005513,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.53062299999874}
{"item_id":"semeval2013.d005.s028.t003","gold_sense_keys":["change%1:04:00::","change%1:11:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"change%1:04:00::","call_ids":["semeval2013.d005.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"change%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008626,"input_uncached_usd":0.000037,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.093314124969766}
{"item_id":"semeval2013.d005.s028.t004","gold_sense_keys":["adaptation%1:22:00::"],"candidates":[{"index":1,"sense_key":"adaptation%1:10:00::","synset_id":"adaptation.n.01"},{"index":2,"sense_key":"adaptation%1:22:00::","synset_id":"adaptation.n.02"},{"index":3,"sense_key":"adaptation%1:04:01::","synset_id":"adaptation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"adaptation%1:22:00::","call_ids":["semeval2013.d005.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"adaptation%1:22:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013084999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.316263707994949}
{"item_id":"semeval2013.d005.s028.t005","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d005.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004277,"input_uncached_usd":0.0000373,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.960621250036638}
{"item_id":"semeval2013.d005.s029.t001","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d005.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005206500000000001,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.147522999963257}
{"item_id":"semeval2013.d005.s030.t000","gold_sense_keys":["filing%1:10:00::"],"candidates":[{"index":1,"sense_key":"filing%1:10:00::","synset_id":"filing.n.01"},{"index":2,"sense_key":"filing%1:17:00::","synset_id":"filing.n.02"},{"index":3,"sense_key":"filing%1:04:01::","synset_id":"filing.n.03"},{"index":4,"sense_key":"filing%1:04:00::","synset_id":"filing.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"filing%1:10:00::","call_ids":["semeval2013.d005.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"filing%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023454999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7550385000067763}
{"item_id":"semeval2013.d005.s030.t001","gold_sense_keys":["supreme_court%1:14:00::"],"candidates":[{"index":1,"sense_key":"supreme_court%1:14:00::","synset_id":"supreme_court.n.01"},{"index":2,"sense_key":"supreme_court%1:14:01::","synset_id":"supreme_court.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"supreme_court%1:14:00::","call_ids":["semeval2013.d005.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"supreme_court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":428,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036059999999999993,"input_uncached_usd":0.0000214,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.786339209007565}
{"item_id":"semeval2013.d005.s030.t002","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d005.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002385,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.523385750013404}
{"item_id":"semeval2013.d005.s030.t003","gold_sense_keys":["conviction%1:04:00::"],"candidates":[{"index":1,"sense_key":"conviction%1:09:00::","synset_id":"conviction.n.01"},{"index":2,"sense_key":"conviction%1:04:00::","synset_id":"conviction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conviction%1:04:00::","call_ids":["semeval2013.d005.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conviction%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038605,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.460476708016358}
{"item_id":"semeval2013.d005.s030.t004","gold_sense_keys":["trial%1:04:02::"],"candidates":[{"index":1,"sense_key":"trial%1:04:00::","synset_id":"test.n.05"},{"index":2,"sense_key":"trial%1:09:00::","synset_id":"trial.n.02"},{"index":3,"sense_key":"trial%1:04:03::","synset_id":"test.n.04"},{"index":4,"sense_key":"trial%1:04:02::","synset_id":"trial.n.04"},{"index":5,"sense_key":"trial%1:11:01::","synset_id":"trial.n.05"},{"index":6,"sense_key":"trial%1:11:00::","synset_id":"trial.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"trial%1:04:02::","call_ids":["semeval2013.d005.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"trial%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024234999999999998,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.007929874991532}
{"item_id":"semeval2013.d005.s030.t005","gold_sense_keys":["coverage%1:10:00::"],"candidates":[{"index":1,"sense_key":"coverage%1:21:00::","synset_id":"coverage.n.01"},{"index":2,"sense_key":"coverage%1:07:00::","synset_id":"coverage.n.02"},{"index":3,"sense_key":"coverage%1:10:00::","synset_id":"coverage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"coverage%1:10:00::","call_ids":["semeval2013.d005.s030.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"coverage%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010529999999999998,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.065662915993016}
{"item_id":"semeval2013.d005.s030.t006","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d005.s030.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00033185,"input_uncached_usd":0.000043849999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.8474726670538075}
{"item_id":"semeval2013.d005.s031.t000","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d005.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1157,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00026904999999999995,"input_uncached_usd":0.00005785,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.490160209010355}
{"item_id":"semeval2013.d005.s031.t001","gold_sense_keys":["century%1:28:00::"],"candidates":[{"index":1,"sense_key":"century%1:28:00::","synset_id":"century.n.01"},{"index":2,"sense_key":"century%1:23:00::","synset_id":"hundred.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"century%1:28:00::","call_ids":["semeval2013.d005.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"century%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":354,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035729999999999996,"input_uncached_usd":0.0000177,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.6039921249612235}
{"item_id":"semeval2013.d006.s000.t001","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":3,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fraud%1:04:01::","call_ids":["semeval2013.d006.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fraud%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":298,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045649999999999993,"input_uncached_usd":0.0000149,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.502898500009906}
{"item_id":"semeval2013.d006.s000.t002","gold_sense_keys":["law%1:14:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"law%1:14:00::","call_ids":["semeval2013.d006.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"law%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046639999999999995,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.31463433400495}
{"item_id":"semeval2013.d006.s001.t000","gold_sense_keys":["trick%1:04:05::"],"candidates":[{"index":1,"sense_key":"trick%1:04:05::","synset_id":"trick.n.01"},{"index":2,"sense_key":"trick%1:28:00::","synset_id":"trick.n.02"},{"index":3,"sense_key":"trick%1:04:04::","synset_id":"trick.n.03"},{"index":4,"sense_key":"trick%1:04:00::","synset_id":"antic.n.01"},{"index":5,"sense_key":"trick%1:04:02::","synset_id":"magic_trick.n.01"},{"index":6,"sense_key":"trick%1:18:00::","synset_id":"whoremaster.n.01"},{"index":7,"sense_key":"trick%1:04:01::","synset_id":"trick.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"trick%1:04:04::","call_ids":["semeval2013.d006.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"trick%1:04:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":2256,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009288,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0009023999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.965798042016104}
{"item_id":"semeval2013.d006.s001.t002","gold_sense_keys":["will%1:09:01::"],"candidates":[{"index":1,"sense_key":"will%1:09:00::","synset_id":"volition.n.01"},{"index":2,"sense_key":"will%1:09:01::","synset_id":"will.n.02"},{"index":3,"sense_key":"will%1:10:00::","synset_id":"will.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"will%1:09:01::","call_ids":["semeval2013.d006.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"will%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030615,"input_uncached_usd":0.00001815,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.302323083975352}
{"item_id":"semeval2013.d006.s002.t000","gold_sense_keys":["expulsion%1:04:01::"],"candidates":[{"index":1,"sense_key":"expulsion%1:04:01::","synset_id":"ejection.n.02"},{"index":2,"sense_key":"expulsion%1:04:02::","synset_id":"extrusion.n.02"},{"index":3,"sense_key":"expulsion%1:04:00::","synset_id":"expulsion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"expulsion%1:04:00::","call_ids":["semeval2013.d006.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"expulsion%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041205,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.460691999993287}
{"item_id":"semeval2013.d006.s002.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010604999999999999,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.146608458016999}
{"item_id":"semeval2013.d006.s002.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d006.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037469999999999995,"input_uncached_usd":0.000035499999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.853828041988891}
{"item_id":"semeval2013.d006.s002.t003","gold_sense_keys":["ink%1:27:01::"],"candidates":[{"index":1,"sense_key":"ink%1:27:01::","synset_id":"ink.n.01"},{"index":2,"sense_key":"ink%1:08:00::","synset_id":"ink.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ink%1:27:01::","call_ids":["semeval2013.d006.s002.t003__v1__a1","semeval2013.d006.s002.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ink%1:27:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045934999999999997,"input_uncached_usd":0.000036950000000000004,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.571892415988259}
{"item_id":"semeval2013.d006.s002.t004","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"behavior%1:07:00::","call_ids":["semeval2013.d006.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"behavior%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025955,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.198840165976435}
{"item_id":"semeval2013.d006.s002.t005","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020845,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9272976249922067}
{"item_id":"semeval2013.d006.s003.t000","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2013.d006.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004193,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.604262791981455}
{"item_id":"semeval2013.d006.s003.t001","gold_sense_keys":["end%1:09:00::","end%1:11:00::","end%1:28:00::"],"candidates":[{"index":1,"sense_key":"end%1:15:00::","synset_id":"end.n.01"},{"index":2,"sense_key":"end%1:28:00::","synset_id":"end.n.02"},{"index":3,"sense_key":"end%1:11:00::","synset_id":"end.n.03"},{"index":4,"sense_key":"end%1:09:02::","synset_id":"goal.n.01"},{"index":5,"sense_key":"end%1:09:00::","synset_id":"end.n.05"},{"index":6,"sense_key":"end%1:26:00::","synset_id":"end.n.06"},{"index":7,"sense_key":"end%1:15:02::","synset_id":"end.n.07"},{"index":8,"sense_key":"end%1:18:00::","synset_id":"end.n.08"},{"index":9,"sense_key":"end%1:15:01::","synset_id":"end.n.09"},{"index":10,"sense_key":"end%1:15:03::","synset_id":"end.n.10"},{"index":11,"sense_key":"end%1:04:00::","synset_id":"end.n.11"},{"index":12,"sense_key":"end%1:10:00::","synset_id":"conclusion.n.08"},{"index":13,"sense_key":"end%1:06:02::","synset_id":"end.n.13"},{"index":14,"sense_key":"end%1:04:02::","synset_id":"end.n.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"end%1:28:00::","call_ids":["semeval2013.d006.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"end%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":928,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002576,"input_uncached_usd":0.000046399999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8572194169973955}
{"item_id":"semeval2013.d006.s003.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d006.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004283,"input_uncached_usd":0.0000379,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.002301207976416}
{"item_id":"semeval2013.d006.s003.t003","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s003.t003__v1__a1","semeval2013.d006.s003.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041254999999999997,"input_uncached_usd":0.00004135,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.717134876002092}
{"item_id":"semeval2013.d006.s003.t004","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":505,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026205,"input_uncached_usd":0.00002525,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.190669874951709}
{"item_id":"semeval2013.d006.s003.t005","gold_sense_keys":["time%1:28:05::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["semeval2013.d006.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040175,"input_uncached_usd":0.00003695,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.066146500001196}
{"item_id":"semeval2013.d006.s003.t006","gold_sense_keys":["referee%1:18:00::"],"candidates":[{"index":1,"sense_key":"referee%1:18:00::","synset_id":"referee.n.01"},{"index":2,"sense_key":"referee%1:18:01::","synset_id":"reviewer.n.01"},{"index":3,"sense_key":"referee%1:18:02::","synset_id":"referee.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"referee%1:18:00::","call_ids":["semeval2013.d006.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"referee%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010534999999999999,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.002888500050176}
{"item_id":"semeval2013.d006.s004.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d006.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007138,"input_uncached_usd":0.0000418,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.849820917006582}
{"item_id":"semeval2013.d006.s004.t002","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d006.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028655,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.674787041032687}
{"item_id":"semeval2013.d006.s004.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031525,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.613707042008173}
{"item_id":"semeval2013.d006.s004.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d006.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":790,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005579000000000001,"input_uncached_usd":0.0000395,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.520290667016525}
{"item_id":"semeval2013.d006.s004.t006","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":3,"sense_key":"league%1:23:00::","synset_id":"league.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d006.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020835,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.134169958007988}
{"item_id":"semeval2013.d006.s004.t008","gold_sense_keys":["phase%1:28:00::"],"candidates":[{"index":1,"sense_key":"phase%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"phase%1:26:00::","synset_id":"phase.n.02"},{"index":3,"sense_key":"phase%1:28:01::","synset_id":"phase.n.03"},{"index":4,"sense_key":"phase%1:07:00::","synset_id":"phase.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phase%1:28:00::","call_ids":["semeval2013.d006.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phase%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031685000000000003,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0873611670103855}
{"item_id":"semeval2013.d006.s005.t000","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2013.d006.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.000414,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.281955832964741}
{"item_id":"semeval2013.d006.s005.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004313,"input_uncached_usd":0.0000409,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.968760291987564}
{"item_id":"semeval2013.d006.s005.t003","gold_sense_keys":["trainer%1:18:00::"],"candidates":[{"index":1,"sense_key":"trainer%1:18:00::","synset_id":"trainer.n.01"},{"index":2,"sense_key":"trainer%1:06:00::","synset_id":"flight_simulator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trainer%1:18:00::","call_ids":["semeval2013.d006.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trainer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005416999999999999,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.465614790969994}
{"item_id":"semeval2013.d006.s005.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048965,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.617990290978923}
{"item_id":"semeval2013.d006.s005.t005","gold_sense_keys":["amount%1:21:00::"],"candidates":[{"index":1,"sense_key":"amount%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"amount%1:07:00::","synset_id":"amount.n.02"},{"index":3,"sense_key":"amount%1:03:00::","synset_id":"measure.n.02"},{"index":4,"sense_key":"amount%1:09:00::","synset_id":"sum.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"amount%1:21:00::","call_ids":["semeval2013.d006.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"amount%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026435,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.964864709007088}
{"item_id":"semeval2013.d006.s005.t006","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000265,"input_uncached_usd":0.000028199999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.921120457991492}
{"item_id":"semeval2013.d006.s006.t000","gold_sense_keys":["provocation%1:16:00::"],"candidates":[{"index":1,"sense_key":"provocation%1:04:00::","synset_id":"aggravation.n.02"},{"index":2,"sense_key":"provocation%1:16:00::","synset_id":"incitement.n.03"},{"index":3,"sense_key":"provocation%1:10:00::","synset_id":"provocation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"provocation%1:16:00::","call_ids":["semeval2013.d006.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"provocation%1:16:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028905,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.595659125014208}
{"item_id":"semeval2013.d006.s006.t002","gold_sense_keys":["circumstance%1:26:01::","circumstance%1:26:02::"],"candidates":[{"index":1,"sense_key":"circumstance%1:26:01::","synset_id":"circumstance.n.01"},{"index":2,"sense_key":"circumstance%1:26:02::","synset_id":"context.n.02"},{"index":3,"sense_key":"circumstance%1:09:00::","synset_id":"circumstance.n.03"},{"index":4,"sense_key":"circumstance%1:11:00::","synset_id":"circumstance.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"circumstance%1:26:02::","call_ids":["semeval2013.d006.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"circumstance%1:26:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":564,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001882,"input_uncached_usd":0.000028199999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5687949580023997}
{"item_id":"semeval2013.d006.s006.t003","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2013.d006.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031915000000000003,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.923801041964907}
{"item_id":"semeval2013.d006.s006.t004","gold_sense_keys":["world%1:14:01::"],"candidates":[{"index":1,"sense_key":"world%1:17:01::","synset_id":"universe.n.01"},{"index":2,"sense_key":"world%1:14:01::","synset_id":"world.n.02"},{"index":3,"sense_key":"world%1:09:00::","synset_id":"world.n.03"},{"index":4,"sense_key":"world%1:17:00::","synset_id":"earth.n.01"},{"index":5,"sense_key":"world%1:14:02::","synset_id":"populace.n.01"},{"index":6,"sense_key":"world%1:17:02::","synset_id":"world.n.06"},{"index":7,"sense_key":"world%1:09:01::","synset_id":"worldly_concern.n.01"},{"index":8,"sense_key":"world%1:05:00::","synset_id":"world.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"world%1:17:02::","call_ids":["semeval2013.d006.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"world%1:17:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030294999999999996,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.786524291965179}
{"item_id":"semeval2013.d006.s006.t005","gold_sense_keys":["football%1:04:00::"],"candidates":[{"index":1,"sense_key":"football%1:04:00::","synset_id":"football.n.01"},{"index":2,"sense_key":"football%1:06:00::","synset_id":"football.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"football%1:04:00::","call_ids":["semeval2013.d006.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"football%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038944999999999995,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.082071667013224}
{"item_id":"semeval2013.d006.s007.t001","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"rule%1:09:01::","call_ids":["semeval2013.d006.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"rule%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1016,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004412,"input_uncached_usd":0.000050799999999999995,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.476951041026041}
{"item_id":"semeval2013.d006.s008.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2013.d006.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1052,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.000443,"input_uncached_usd":0.0000526,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.492553416988812}
{"item_id":"semeval2013.d006.s008.t001","gold_sense_keys":["debate%1:10:01::"],"candidates":[{"index":1,"sense_key":"debate%1:10:01::","synset_id":"argument.n.03"},{"index":2,"sense_key":"debate%1:10:00::","synset_id":"debate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"debate%1:10:01::","call_ids":["semeval2013.d006.s008.t001__v1__a1","semeval2013.d006.s008.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"debate%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1157,"cached_input_tokens":0,"output_tokens":1377,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00060865,"input_uncached_usd":0.00005785,"input_cached_usd":0.0,"output_usd":0.0005507999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.380671499005985}
{"item_id":"semeval2013.d006.s008.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d006.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1321,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00068685,"input_uncached_usd":0.00006604999999999999,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.69418983295327}
{"item_id":"semeval2013.d006.s008.t003","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fact%1:26:01::","call_ids":["semeval2013.d006.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fact%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049885,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.064845791028347}
{"item_id":"semeval2013.d006.s008.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s008.t004__v1__a1","semeval2013.d006.s008.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1077,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006298499999999999,"input_uncached_usd":0.000053849999999999994,"input_cached_usd":0.0,"output_usd":0.0005759999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.849541041941848}
{"item_id":"semeval2013.d006.s008.t005","gold_sense_keys":["organization%1:14:00::","organization%1:14:01::"],"candidates":[{"index":1,"sense_key":"organization%1:14:00::","synset_id":"organization.n.01"},{"index":2,"sense_key":"organization%1:09:00::","synset_id":"arrangement.n.03"},{"index":3,"sense_key":"organization%1:14:01::","synset_id":"administration.n.02"},{"index":4,"sense_key":"organization%1:04:00::","synset_id":"organization.n.04"},{"index":5,"sense_key":"organization%1:07:00::","synset_id":"organization.n.05"},{"index":6,"sense_key":"organization%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organization%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"organization%1:14:01::","call_ids":["semeval2013.d006.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"organization%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027854999999999997,"input_uncached_usd":0.00004175,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.163621750019956}
{"item_id":"semeval2013.d006.s008.t007","gold_sense_keys":["nobility%1:07:00::"],"candidates":[{"index":1,"sense_key":"nobility%1:14:00::","synset_id":"nobility.n.01"},{"index":2,"sense_key":"nobility%1:07:00::","synset_id":"nobility.n.02"},{"index":3,"sense_key":"nobility%1:26:00::","synset_id":"nobility.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nobility%1:07:00::","call_ids":["semeval2013.d006.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nobility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049555,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.185006457963027}
{"item_id":"semeval2013.d006.s008.t008","gold_sense_keys":["honor%1:07:00::"],"candidates":[{"index":1,"sense_key":"honor%1:10:00::","synset_id":"award.n.02"},{"index":2,"sense_key":"honor%1:26:00::","synset_id":"honor.n.02"},{"index":3,"sense_key":"honor%1:07:00::","synset_id":"honor.n.03"},{"index":4,"sense_key":"honor%1:07:01::","synset_id":"honor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"honor%1:07:00::","call_ids":["semeval2013.d006.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"honor%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001135,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4276547079789452}
{"item_id":"semeval2013.d006.s008.t009","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001147,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7449806249933317}
{"item_id":"semeval2013.d006.s008.t010","gold_sense_keys":["act%1:03:00::"],"candidates":[{"index":1,"sense_key":"act%1:10:01::","synset_id":"act.n.01"},{"index":2,"sense_key":"act%1:03:00::","synset_id":"act.n.02"},{"index":3,"sense_key":"act%1:10:00::","synset_id":"act.n.03"},{"index":4,"sense_key":"act%1:10:02::","synset_id":"act.n.04"},{"index":5,"sense_key":"act%1:10:03::","synset_id":"act.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"act%1:03:00::","call_ids":["semeval2013.d006.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"act%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021794999999999999,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8088942500180565}
{"item_id":"semeval2013.d006.s008.t012","gold_sense_keys":["history%1:28:00::"],"candidates":[{"index":1,"sense_key":"history%1:28:00::","synset_id":"history.n.01"},{"index":2,"sense_key":"history%1:10:00::","synset_id":"history.n.02"},{"index":3,"sense_key":"history%1:09:00::","synset_id":"history.n.03"},{"index":4,"sense_key":"history%1:28:02::","synset_id":"history.n.04"},{"index":5,"sense_key":"history%1:09:01::","synset_id":"history.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"history%1:10:00::","call_ids":["semeval2013.d006.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"history%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003217,"input_uncached_usd":0.0000337,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.342984542017803}
{"item_id":"semeval2013.d006.s009.t000","gold_sense_keys":["debate%1:10:01::"],"candidates":[{"index":1,"sense_key":"debate%1:10:01::","synset_id":"argument.n.03"},{"index":2,"sense_key":"debate%1:10:00::","synset_id":"debate.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"debate%1:10:01::","call_ids":["semeval2013.d006.s009.t000__v1__a1","semeval2013.d006.s009.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"debate%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00052885,"input_uncached_usd":0.00005525,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.579058875038754}
{"item_id":"semeval2013.d006.s009.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":670,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021909999999999999,"input_uncached_usd":0.0000335,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.62469179200707}
{"item_id":"semeval2013.d006.s009.t002","gold_sense_keys":["behavior%1:04:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"behavior%1:04:00::","call_ids":["semeval2013.d006.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"behavior%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006253000000000001,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.275848874996882}
{"item_id":"semeval2013.d006.s009.t003","gold_sense_keys":["principle%1:09:01::"],"candidates":[{"index":1,"sense_key":"principle%1:09:03::","synset_id":"principle.n.01"},{"index":2,"sense_key":"principle%1:09:01::","synset_id":"principle.n.02"},{"index":3,"sense_key":"principle%1:09:00::","synset_id":"principle.n.03"},{"index":4,"sense_key":"principle%1:09:04::","synset_id":"principle.n.04"},{"index":5,"sense_key":"principle%1:10:00::","synset_id":"principle.n.05"},{"index":6,"sense_key":"principle%1:09:02::","synset_id":"rationale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"principle%1:09:01::","call_ids":["semeval2013.d006.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"principle%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":700,"cached_input_tokens":0,"output_tokens":1873,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007841999999999999,"input_uncached_usd":0.000035,"input_cached_usd":0.0,"output_usd":0.0007492,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.217965041985735}
{"item_id":"semeval2013.d006.s009.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005888999999999998,"input_uncached_usd":0.0000445,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.182180291973054}
{"item_id":"semeval2013.d006.s009.t006","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":3,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fraud%1:04:01::","call_ids":["semeval2013.d006.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fraud%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016204999999999998,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.182817165972665}
{"item_id":"semeval2013.d006.s009.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004093,"input_uncached_usd":0.0000445,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.40086983301444}
{"item_id":"semeval2013.d006.s009.t009","gold_sense_keys":["regulation%1:10:00::"],"candidates":[{"index":1,"sense_key":"regulation%1:10:00::","synset_id":"regulation.n.01"},{"index":2,"sense_key":"regulation%1:09:00::","synset_id":"rule.n.01"},{"index":3,"sense_key":"regulation%1:26:00::","synset_id":"regulation.n.03"},{"index":4,"sense_key":"regulation%1:22:00::","synset_id":"regulation.n.04"},{"index":5,"sense_key":"regulation%1:04:01::","synset_id":"regulation.n.05"},{"index":6,"sense_key":"regulation%1:04:00::","synset_id":"regulation.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"regulation%1:10:00::","call_ids":["semeval2013.d006.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"regulation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004231,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.743567916972097}
{"item_id":"semeval2013.d006.s010.t000","gold_sense_keys":["matter%1:09:00::"],"candidates":[{"index":1,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":2,"sense_key":"matter%1:09:00::","synset_id":"topic.n.02"},{"index":3,"sense_key":"matter%1:03:00::","synset_id":"matter.n.03"},{"index":4,"sense_key":"matter%1:09:02::","synset_id":"matter.n.04"},{"index":5,"sense_key":"matter%1:07:00::","synset_id":"matter.n.05"},{"index":6,"sense_key":"matter%1:10:00::","synset_id":"matter.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"matter%1:09:00::","call_ids":["semeval2013.d006.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"matter%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032125000000000003,"input_uncached_usd":0.00003325,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.692890750011429}
{"item_id":"semeval2013.d006.s010.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016744999999999998,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.310758999956306}
{"item_id":"semeval2013.d006.s010.t002","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"damage%1:04:00::","call_ids":["semeval2013.d006.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"damage%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013864999999999998,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.652861542010214}
{"item_id":"semeval2013.d006.s011.t000","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00069945,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.887541499978397}
{"item_id":"semeval2013.d006.s011.t003","gold_sense_keys":["integrity%1:26:00::"],"candidates":[{"index":1,"sense_key":"integrity%1:26:00::","synset_id":"integrity.n.01"},{"index":2,"sense_key":"integrity%1:07:00::","synset_id":"integrity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"integrity%1:26:00::","call_ids":["semeval2013.d006.s011.t003__v1__a1","semeval2013.d006.s011.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"integrity%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1141,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004794499999999999,"input_uncached_usd":0.00005705,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.666267250024248}
{"item_id":"semeval2013.d006.s011.t004","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019015,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.052491500042379}
{"item_id":"semeval2013.d006.s011.t005","gold_sense_keys":["aspect%1:07:02::"],"candidates":[{"index":1,"sense_key":"aspect%1:09:00::","synset_id":"aspect.n.01"},{"index":2,"sense_key":"aspect%1:07:02::","synset_id":"aspect.n.02"},{"index":3,"sense_key":"aspect%1:09:01::","synset_id":"view.n.02"},{"index":4,"sense_key":"aspect%1:24:00::","synset_id":"aspect.n.04"},{"index":5,"sense_key":"aspect%1:07:01::","synset_id":"expression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"aspect%1:07:02::","call_ids":["semeval2013.d006.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"aspect%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006519999999999999,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.57510945800459}
{"item_id":"semeval2013.d006.s011.t006","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":890,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004349,"input_uncached_usd":0.0000445,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.101277541019954}
{"item_id":"semeval2013.d006.s011.t007","gold_sense_keys":["danger%1:26:00::"],"candidates":[{"index":1,"sense_key":"danger%1:26:00::","synset_id":"danger.n.01"},{"index":2,"sense_key":"danger%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"danger%1:26:01::","synset_id":"danger.n.03"},{"index":4,"sense_key":"danger%1:15:00::","synset_id":"danger.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"danger%1:26:00::","call_ids":["semeval2013.d006.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"danger%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":2960,"reasoning_output_tokens":2944},"cost":{"currency":"USD","total_usd":0.0012135499999999999,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.001184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.869018291996326}
{"item_id":"semeval2013.d006.s012.t000","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002148,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.131707916036248}
{"item_id":"semeval2013.d006.s012.t001","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003142,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.036558750027325}
{"item_id":"semeval2013.d006.s012.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042235,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.42271349998191}
{"item_id":"semeval2013.d006.s012.t004","gold_sense_keys":["take_advantage%2:40:00::"],"candidates":[{"index":1,"sense_key":"take_advantage%2:40:00::","synset_id":"capitalize.v.01"},{"index":2,"sense_key":"take_advantage%2:41:12::","synset_id":"trespass.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"take_advantage%2:40:00::","call_ids":["semeval2013.d006.s012.t004__v1__a1","semeval2013.d006.s012.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"take_advantage%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1185,"cached_input_tokens":0,"output_tokens":2592,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010960499999999999,"input_uncached_usd":0.00005925,"input_cached_usd":0.0,"output_usd":0.0010368,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.995343208953273}
{"item_id":"semeval2013.d006.s012.t005","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026294999999999997,"input_uncached_usd":0.00002615,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.625834750011563}
{"item_id":"semeval2013.d006.s013.t000","gold_sense_keys":["thing%1:07:00::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"thing%1:26:00::","call_ids":["semeval2013.d006.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"thing%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006378,"input_uncached_usd":0.0000426,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.047377165989019}
{"item_id":"semeval2013.d006.s013.t001","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043599999999999997,"input_uncached_usd":0.0000456,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.501370707992464}
{"item_id":"semeval2013.d006.s013.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026809999999999996,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.969650625018403}
{"item_id":"semeval2013.d006.s013.t005","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"rule%1:10:02::","call_ids":["semeval2013.d006.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"rule%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1006,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005175,"input_uncached_usd":0.000050299999999999996,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.668213166005444}
{"item_id":"semeval2013.d006.s013.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":912,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035919999999999995,"input_uncached_usd":0.0000456,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.328939082974102}
{"item_id":"semeval2013.d006.s013.t009","gold_sense_keys":["referee%1:18:00::"],"candidates":[{"index":1,"sense_key":"referee%1:18:00::","synset_id":"referee.n.01"},{"index":2,"sense_key":"referee%1:18:01::","synset_id":"reviewer.n.01"},{"index":3,"sense_key":"referee%1:18:02::","synset_id":"referee.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"referee%1:18:00::","call_ids":["semeval2013.d006.s013.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"referee%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021375,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.255846291023772}
{"item_id":"semeval2013.d006.s014.t001","gold_sense_keys":["complaint%1:10:00::"],"candidates":[{"index":1,"sense_key":"complaint%1:26:00::","synset_id":"ailment.n.01"},{"index":2,"sense_key":"complaint%1:10:02::","synset_id":"complaint.n.02"},{"index":3,"sense_key":"complaint%1:10:00::","synset_id":"complaint.n.03"},{"index":4,"sense_key":"complaint%1:10:03::","synset_id":"complaint.n.04"},{"index":5,"sense_key":"complaint%1:10:04::","synset_id":"charge.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"complaint%1:10:00::","call_ids":["semeval2013.d006.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"complaint%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000863,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.865884124999866}
{"item_id":"semeval2013.d006.s015.t000","gold_sense_keys":["question%1:10:00::","question%1:10:01::","question%1:10:03::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d006.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":671,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039834999999999995,"input_uncached_usd":0.000033549999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.829889708023984}
{"item_id":"semeval2013.d006.s015.t001","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d006.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002163,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8809984590043314}
{"item_id":"semeval2013.d006.s015.t002","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004151,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5464460420189425}
{"item_id":"semeval2013.d006.s015.t004","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040649999999999996,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.152211125008762}
{"item_id":"semeval2013.d006.s016.t000","gold_sense_keys":["situation%1:26:00::","situation%1:26:01::"],"candidates":[{"index":1,"sense_key":"situation%1:26:00::","synset_id":"situation.n.01"},{"index":2,"sense_key":"situation%1:26:01::","synset_id":"situation.n.02"},{"index":3,"sense_key":"situation%1:26:02::","synset_id":"situation.n.03"},{"index":4,"sense_key":"situation%1:15:00::","synset_id":"site.n.02"},{"index":5,"sense_key":"situation%1:04:00::","synset_id":"position.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"situation%1:26:00::","call_ids":["semeval2013.d006.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"situation%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005503999999999999,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.546849875012413}
{"item_id":"semeval2013.d006.s016.t002","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2013.d006.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056775,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.301338499994017}
{"item_id":"semeval2013.d006.s016.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d006.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003285,"input_uncached_usd":0.000040499999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.459570707986131}
{"item_id":"semeval2013.d006.s016.t004","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["semeval2013.d006.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042025,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.2791136669693515}
{"item_id":"semeval2013.d006.s016.t005","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"action%1:04:03::","call_ids":["semeval2013.d006.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"action%1:04:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007141999999999999,"input_uncached_usd":0.000042199999999999996,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.993747457978316}
{"item_id":"semeval2013.d006.s016.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059175,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.8430846669944}
{"item_id":"semeval2013.d006.s016.t007","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":557,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004950499999999999,"input_uncached_usd":0.00002785,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.146064916043542}
{"item_id":"semeval2013.d006.s017.t000","gold_sense_keys":["strategy%1:09:00::"],"candidates":[{"index":1,"sense_key":"strategy%1:09:00::","synset_id":"scheme.n.01"},{"index":2,"sense_key":"strategy%1:09:01::","synset_id":"strategy.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"strategy%1:09:00::","call_ids":["semeval2013.d006.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"strategy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003357,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.359861707955133}
{"item_id":"semeval2013.d006.s017.t001","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037935,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.413124832964968}
{"item_id":"semeval2013.d006.s017.t003","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006483,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.04223008401459}
{"item_id":"semeval2013.d006.s018.t000","gold_sense_keys":["advantage%1:07:00::","advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036529999999999993,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.315461416030303}
{"item_id":"semeval2013.d006.s018.t001","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fact%1:09:01::","call_ids":["semeval2013.d006.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fact%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":580,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005986000000000001,"input_uncached_usd":0.000029,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.719105875003152}
{"item_id":"semeval2013.d006.s018.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021444999999999998,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6434332909993827}
{"item_id":"semeval2013.d006.s018.t003","gold_sense_keys":["phase%1:28:00::"],"candidates":[{"index":1,"sense_key":"phase%1:28:00::","synset_id":"phase.n.01"},{"index":2,"sense_key":"phase%1:26:00::","synset_id":"phase.n.02"},{"index":3,"sense_key":"phase%1:28:01::","synset_id":"phase.n.03"},{"index":4,"sense_key":"phase%1:07:00::","synset_id":"phase.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phase%1:28:00::","call_ids":["semeval2013.d006.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phase%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006261000000000001,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.461531083041336}
{"item_id":"semeval2013.d006.s018.t004","gold_sense_keys":["championship%1:11:00::"],"candidates":[{"index":1,"sense_key":"championship%1:26:00::","synset_id":"championship.n.01"},{"index":2,"sense_key":"championship%1:11:00::","synset_id":"championship.n.02"},{"index":3,"sense_key":"championship%1:04:00::","synset_id":"backing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"championship%1:11:00::","call_ids":["semeval2013.d006.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"championship%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041645,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.397868916974403}
{"item_id":"semeval2013.d006.s018.t006","gold_sense_keys":["participation%1:04:00::"],"candidates":[{"index":1,"sense_key":"participation%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"participation%1:26:00::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participation%1:04:00::","call_ids":["semeval2013.d006.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004921,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.848818707978353}
{"item_id":"semeval2013.d006.s018.t007","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032945,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.038844124996103}
{"item_id":"semeval2013.d006.s019.t000","gold_sense_keys":["term%1:28:00::"],"candidates":[{"index":1,"sense_key":"term%1:10:00::","synset_id":"term.n.01"},{"index":2,"sense_key":"term%1:28:00::","synset_id":"term.n.02"},{"index":3,"sense_key":"term%1:10:02::","synset_id":"condition.n.07"},{"index":4,"sense_key":"term%1:09:00::","synset_id":"term.n.04"},{"index":5,"sense_key":"term%1:10:01::","synset_id":"term.n.05"},{"index":6,"sense_key":"term%1:28:01::","synset_id":"term.n.06"},{"index":7,"sense_key":"term%1:06:00::","synset_id":"terminus.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"term%1:28:00::","call_ids":["semeval2013.d006.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"term%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":662,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034669999999999997,"input_uncached_usd":0.0000331,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.356940332974773}
{"item_id":"semeval2013.d006.s019.t001","gold_sense_keys":["advantage%1:07:00::","advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018385,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3620812919689342}
{"item_id":"semeval2013.d006.s019.t002","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"degree%1:07:00::","call_ids":["semeval2013.d006.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047484999999999997,"input_uncached_usd":0.00003325,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.656637334031984}
{"item_id":"semeval2013.d006.s019.t004","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d006.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030885,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.458629833010491}
{"item_id":"semeval2013.d006.s019.t005","gold_sense_keys":["change%1:11:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%1:11:00::","call_ids":["semeval2013.d006.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":787,"cached_input_tokens":0,"output_tokens":3218,"reasoning_output_tokens":3200},"cost":{"currency":"USD","total_usd":0.00132655,"input_uncached_usd":0.00003935,"input_cached_usd":0.0,"output_usd":0.0012871999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.03306537499884}
{"item_id":"semeval2013.d006.s019.t007","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005849,"input_uncached_usd":0.0000409,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.597478417039383}
{"item_id":"semeval2013.d006.s019.t008","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005962000000000001,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.526033584028482}
{"item_id":"semeval2013.d006.s019.t010","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d006.s019.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048084999999999995,"input_uncached_usd":0.00003925,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.502272749959957}
{"item_id":"semeval2013.d006.s019.t013","gold_sense_keys":["suspension%1:04:01::"],"candidates":[{"index":1,"sense_key":"suspension%1:27:00::","synset_id":"suspension.n.01"},{"index":2,"sense_key":"suspension%1:28:00::","synset_id":"pause.n.01"},{"index":3,"sense_key":"suspension%1:26:00::","synset_id":"abeyance.n.01"},{"index":4,"sense_key":"suspension%1:11:00::","synset_id":"suspension.n.04"},{"index":5,"sense_key":"suspension%1:06:00::","synset_id":"suspension.n.05"},{"index":6,"sense_key":"suspension%1:04:00::","synset_id":"suspension.n.06"},{"index":7,"sense_key":"suspension%1:04:01::","synset_id":"suspension.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"suspension%1:04:01::","call_ids":["semeval2013.d006.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"suspension%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024324999999999998,"input_uncached_usd":0.00003205,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.182371083996259}
{"item_id":"semeval2013.d006.s019.t015","gold_sense_keys":["accumulation%1:04:00::"],"candidates":[{"index":1,"sense_key":"accumulation%1:22:00::","synset_id":"accretion.n.01"},{"index":2,"sense_key":"accumulation%1:14:00::","synset_id":"collection.n.01"},{"index":3,"sense_key":"accumulation%1:04:00::","synset_id":"accumulation.n.03"},{"index":4,"sense_key":"accumulation%1:21:00::","synset_id":"accumulation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"accumulation%1:04:00::","call_ids":["semeval2013.d006.s019.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"accumulation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013394999999999998,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1344786670524627}
{"item_id":"semeval2013.d006.s020.t000","gold_sense_keys":["round%1:28:01::"],"candidates":[{"index":1,"sense_key":"round%1:06:01::","synset_id":"round.n.01"},{"index":2,"sense_key":"round%1:28:01::","synset_id":"cycle.n.01"},{"index":3,"sense_key":"round%1:15:00::","synset_id":"beat.n.01"},{"index":4,"sense_key":"round%1:04:00::","synset_id":"round.n.04"},{"index":5,"sense_key":"round%1:04:03::","synset_id":"round_of_golf.n.01"},{"index":6,"sense_key":"round%1:04:01::","synset_id":"round.n.06"},{"index":7,"sense_key":"round%1:28:00::","synset_id":"turn.n.09"},{"index":8,"sense_key":"round%1:17:00::","synset_id":"round.n.08"},{"index":9,"sense_key":"round%1:13:01::","synset_id":"round.n.09"},{"index":10,"sense_key":"round%1:13:00::","synset_id":"round.n.10"},{"index":11,"sense_key":"round%1:10:00::","synset_id":"round.n.11"},{"index":12,"sense_key":"round%1:10:01::","synset_id":"round.n.12"},{"index":13,"sense_key":"round%1:06:03::","synset_id":"rung.n.01"},{"index":14,"sense_key":"round%1:06:00::","synset_id":"circle.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"round%1:28:01::","call_ids":["semeval2013.d006.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"round%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":921,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048764999999999995,"input_uncached_usd":0.00004605,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.456538582977373}
{"item_id":"semeval2013.d006.s020.t001","gold_sense_keys":["advantage%1:07:01::"],"candidates":[{"index":1,"sense_key":"advantage%1:07:00::","synset_id":"advantage.n.01"},{"index":2,"sense_key":"advantage%1:23:00::","synset_id":"advantage.n.02"},{"index":3,"sense_key":"advantage%1:07:01::","synset_id":"advantage.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advantage%1:07:01::","call_ids":["semeval2013.d006.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advantage%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021014999999999999,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.738803750020452}
{"item_id":"semeval2013.d006.s021.t000","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006889999999999999,"input_uncached_usd":0.0000426,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.179505667008925}
{"item_id":"semeval2013.d006.s021.t002","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2013.d006.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":988,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005166,"input_uncached_usd":0.000049399999999999995,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.897005208011251}
{"item_id":"semeval2013.d006.s021.t003","gold_sense_keys":["decision%1:04:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d006.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":615,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042115,"input_uncached_usd":0.000030749999999999995,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.528265208005905}
{"item_id":"semeval2013.d006.s021.t005","gold_sense_keys":["rule%1:10:02::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"rule%1:10:02::","call_ids":["semeval2013.d006.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"rule%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00036084999999999996,"input_uncached_usd":0.000047249999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.532494833983947}
{"item_id":"semeval2013.d006.s022.t000","gold_sense_keys":["assumption%1:09:00::"],"candidates":[{"index":1,"sense_key":"assumption%1:10:00::","synset_id":"premise.n.01"},{"index":2,"sense_key":"assumption%1:09:00::","synset_id":"assumption.n.02"},{"index":3,"sense_key":"assumption%1:04:00::","synset_id":"assumption.n.03"},{"index":4,"sense_key":"assumption%1:28:00::","synset_id":"assumption.n.04"},{"index":5,"sense_key":"assumption%1:11:00::","synset_id":"assumption.n.05"},{"index":6,"sense_key":"assumption%1:07:00::","synset_id":"presumption.n.03"},{"index":7,"sense_key":"assumption%1:04:01::","synset_id":"assumption.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assumption%1:10:00::","call_ids":["semeval2013.d006.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assumption%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006824,"input_uncached_usd":0.000036,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.339202417002525}
{"item_id":"semeval2013.d006.s022.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d006.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044269999999999997,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.9696898750262335}
{"item_id":"semeval2013.d006.s022.t002","gold_sense_keys":["application%1:04:00::"],"candidates":[{"index":1,"sense_key":"application%1:04:02::","synset_id":"application.n.01"},{"index":2,"sense_key":"application%1:10:00::","synset_id":"application.n.02"},{"index":3,"sense_key":"application%1:04:03::","synset_id":"application.n.03"},{"index":4,"sense_key":"application%1:10:01::","synset_id":"application.n.04"},{"index":5,"sense_key":"application%1:06:00::","synset_id":"lotion.n.02"},{"index":6,"sense_key":"application%1:04:01::","synset_id":"application.n.06"},{"index":7,"sense_key":"application%1:04:00::","synset_id":"application.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"application%1:04:00::","call_ids":["semeval2013.d006.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"application%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044994999999999996,"input_uncached_usd":0.00003395,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.099658916005865}
{"item_id":"semeval2013.d006.s022.t004","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d006.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1225,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00060525,"input_uncached_usd":0.00006125,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.914169250056148}
{"item_id":"semeval2013.d006.s023.t000","gold_sense_keys":["decision%1:04:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d006.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005989500000000001,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.394887499976903}
{"item_id":"semeval2013.d006.s024.t002","gold_sense_keys":["law%1:14:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"law%1:14:00::","call_ids":["semeval2013.d006.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"law%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005527,"input_uncached_usd":0.0000343,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.351886374992318}
{"item_id":"semeval2013.d006.s024.t003","gold_sense_keys":["fraud%1:04:01::"],"candidates":[{"index":1,"sense_key":"fraud%1:04:00::","synset_id":"fraud.n.01"},{"index":2,"sense_key":"fraud%1:18:00::","synset_id":"imposter.n.01"},{"index":3,"sense_key":"fraud%1:04:01::","synset_id":"fraud.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fraud%1:04:01::","call_ids":["semeval2013.d006.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fraud%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044039999999999997,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.088756542012561}
{"item_id":"semeval2013.d006.s024.t004","gold_sense_keys":["occasion%1:11:00::"],"candidates":[{"index":1,"sense_key":"occasion%1:11:00::","synset_id":"juncture.n.01"},{"index":2,"sense_key":"occasion%1:11:01::","synset_id":"affair.n.03"},{"index":3,"sense_key":"occasion%1:16:00::","synset_id":"occasion.n.03"},{"index":4,"sense_key":"occasion%1:28:00::","synset_id":"occasion.n.04"},{"index":5,"sense_key":"occasion%1:26:00::","synset_id":"occasion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"occasion%1:28:00::","call_ids":["semeval2013.d006.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"occasion%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007005,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.5091373330215}
{"item_id":"semeval2013.d006.s024.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d006.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1230,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001959,"input_uncached_usd":0.00006149999999999999,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8610915419994853}
{"item_id":"semeval2013.d006.s025.t001","gold_sense_keys":["factor%1:11:00::"],"candidates":[{"index":1,"sense_key":"factor%1:11:00::","synset_id":"factor.n.01"},{"index":2,"sense_key":"factor%1:09:00::","synset_id":"component.n.01"},{"index":3,"sense_key":"factor%1:23:00::","synset_id":"divisor.n.01"},{"index":4,"sense_key":"factor%1:18:00::","synset_id":"agent.n.04"},{"index":5,"sense_key":"factor%1:23:01::","synset_id":"factor.n.05"},{"index":6,"sense_key":"factor%1:09:01::","synset_id":"factor.n.06"},{"index":7,"sense_key":"factor%1:08:00::","synset_id":"gene.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"factor%1:11:00::","call_ids":["semeval2013.d006.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"factor%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":627,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052415,"input_uncached_usd":0.000031349999999999996,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.012456625001505}
{"item_id":"semeval2013.d006.s025.t002","gold_sense_keys":["expulsion%1:04:01::"],"candidates":[{"index":1,"sense_key":"expulsion%1:04:01::","synset_id":"ejection.n.02"},{"index":2,"sense_key":"expulsion%1:04:02::","synset_id":"extrusion.n.02"},{"index":3,"sense_key":"expulsion%1:04:00::","synset_id":"expulsion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"expulsion%1:04:00::","call_ids":["semeval2013.d006.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"expulsion%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046549999999999993,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.436734249989968}
{"item_id":"semeval2013.d006.s025.t003","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d006.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":793,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019965,"input_uncached_usd":0.00003965,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.611500291968696}
{"item_id":"semeval2013.d006.s025.t004","gold_sense_keys":["transmission%1:04:01::"],"candidates":[{"index":1,"sense_key":"transmission%1:04:01::","synset_id":"transmission.n.01"},{"index":2,"sense_key":"transmission%1:10:00::","synset_id":"transmission.n.02"},{"index":3,"sense_key":"transmission%1:23:00::","synset_id":"transmittance.n.01"},{"index":4,"sense_key":"transmission%1:11:00::","synset_id":"infection.n.04"},{"index":5,"sense_key":"transmission%1:06:01::","synset_id":"transmission.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"transmission%1:04:01::","call_ids":["semeval2013.d006.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"transmission%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038954999999999995,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.310439832974225}
{"item_id":"semeval2013.d006.s026.t000","gold_sense_keys":["damage%1:11:00::"],"candidates":[{"index":1,"sense_key":"damage%1:11:00::","synset_id":"damage.n.01"},{"index":2,"sense_key":"damage%1:11:01::","synset_id":"damage.n.02"},{"index":3,"sense_key":"damage%1:04:00::","synset_id":"damage.n.03"},{"index":4,"sense_key":"damage%1:21:00::","synset_id":"price.n.02"},{"index":5,"sense_key":"damage%1:04:01::","synset_id":"wrong.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"damage%1:11:00::","call_ids":["semeval2013.d006.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"damage%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046544999999999996,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.243836125009693}
{"item_id":"semeval2013.d006.s026.t001","gold_sense_keys":["principle%1:09:00::"],"candidates":[{"index":1,"sense_key":"principle%1:09:03::","synset_id":"principle.n.01"},{"index":2,"sense_key":"principle%1:09:01::","synset_id":"principle.n.02"},{"index":3,"sense_key":"principle%1:09:00::","synset_id":"principle.n.03"},{"index":4,"sense_key":"principle%1:09:04::","synset_id":"principle.n.04"},{"index":5,"sense_key":"principle%1:10:00::","synset_id":"principle.n.05"},{"index":6,"sense_key":"principle%1:09:02::","synset_id":"rationale.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"principle%1:09:01::","call_ids":["semeval2013.d006.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"principle%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049575,"input_uncached_usd":0.00002855,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.94224987499183}
{"item_id":"semeval2013.d006.s026.t003","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d006.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058205,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.37086275004549}
{"item_id":"semeval2013.d007.s000.t001","gold_sense_keys":["possibility%1:26:00::"],"candidates":[{"index":1,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":2,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":3,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possibility%1:09:03::","call_ids":["semeval2013.d007.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possibility%1:09:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020519999999999997,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.349219375057146}
{"item_id":"semeval2013.d007.s000.t002","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2013.d007.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030985,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.413367667002603}
{"item_id":"semeval2013.d007.s001.t000","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["semeval2013.d007.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":358,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001523,"input_uncached_usd":0.000017899999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.250312832999043}
{"item_id":"semeval2013.d007.s001.t002","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":364,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004854,"input_uncached_usd":0.0000182,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.680527207965497}
{"item_id":"semeval2013.d007.s001.t003","gold_sense_keys":["nutrient%1:03:00::"],"candidates":[{"index":1,"sense_key":"nutrient%1:03:00::","synset_id":"food.n.01"},{"index":2,"sense_key":"nutrient%1:03:01::","synset_id":"nutrient.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"nutrient%1:03:01::","call_ids":["semeval2013.d007.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"nutrient%1:03:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":314,"cached_input_tokens":0,"output_tokens":850,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035569999999999997,"input_uncached_usd":0.0000157,"input_cached_usd":0.0,"output_usd":0.00033999999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.2823373330174945}
{"item_id":"semeval2013.d007.s001.t004","gold_sense_keys":["scope%1:07:00::"],"candidates":[{"index":1,"sense_key":"scope%1:07:00::","synset_id":"scope.n.01"},{"index":2,"sense_key":"scope%1:26:00::","synset_id":"setting.n.02"},{"index":3,"sense_key":"scope%1:06:02::","synset_id":"telescope.n.01"},{"index":4,"sense_key":"scope%1:06:01::","synset_id":"oscilloscope.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scope%1:07:00::","call_ids":["semeval2013.d007.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scope%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002828,"input_uncached_usd":0.000020399999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.802206333028153}
{"item_id":"semeval2013.d007.s001.t005","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%1:09:01::","call_ids":["semeval2013.d007.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":961,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00031085,"input_uncached_usd":0.000048049999999999996,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.419906208990142}
{"item_id":"semeval2013.d007.s002.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036744999999999996,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.118147125001997}
{"item_id":"semeval2013.d007.s002.t003","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043055000000000003,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.261097165988758}
{"item_id":"semeval2013.d007.s002.t004","gold_sense_keys":["possibility%1:09:03::"],"candidates":[{"index":1,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":2,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":3,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"possibility%1:09:00::","call_ids":["semeval2013.d007.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"possibility%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043735,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.505660334019922}
{"item_id":"semeval2013.d007.s002.t005","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018769999999999998,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.848914834030438}
{"item_id":"semeval2013.d007.s003.t000","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004064,"input_uncached_usd":0.000016,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.023879458021838}
{"item_id":"semeval2013.d007.s003.t001","gold_sense_keys":["capability%1:07:00::"],"candidates":[{"index":1,"sense_key":"capability%1:07:00::","synset_id":"capability.n.01"},{"index":2,"sense_key":"capability%1:26:00::","synset_id":"capability.n.02"},{"index":3,"sense_key":"capability%1:09:00::","synset_id":"capability.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"capability%1:07:00::","call_ids":["semeval2013.d007.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"capability%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045994999999999993,"input_uncached_usd":0.00001835,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.871220082975924}
{"item_id":"semeval2013.d007.s004.t000","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003083,"input_uncached_usd":0.0000203,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.068703458004165}
{"item_id":"semeval2013.d007.s004.t001","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020724999999999997,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.889177832985297}
{"item_id":"semeval2013.d007.s005.t000","gold_sense_keys":["form%1:09:01::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%1:09:01::","call_ids":["semeval2013.d007.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1053,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00028944999999999996,"input_uncached_usd":0.00005265,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.009315166971646}
{"item_id":"semeval2013.d007.s005.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"life%1:03:00::","call_ids":["semeval2013.d007.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"life%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":888,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038359999999999995,"input_uncached_usd":0.000044399999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.614621292043012}
{"item_id":"semeval2013.d007.s005.t002","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024344999999999998,"input_uncached_usd":0.00003225,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.34119029098656}
{"item_id":"semeval2013.d007.s005.t004","gold_sense_keys":["edition%1:10:02::"],"candidates":[{"index":1,"sense_key":"edition%1:10:02::","synset_id":"edition.n.01"},{"index":2,"sense_key":"edition%1:14:00::","synset_id":"edition.n.02"},{"index":3,"sense_key":"edition%1:10:00::","synset_id":"edition.n.03"},{"index":4,"sense_key":"edition%1:09:00::","synset_id":"version.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"edition%1:10:02::","call_ids":["semeval2013.d007.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"edition%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003129,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.867137875000481}
{"item_id":"semeval2013.d007.s005.t005","gold_sense_keys":["journal%1:10:00::"],"candidates":[{"index":1,"sense_key":"journal%1:10:01::","synset_id":"diary.n.01"},{"index":2,"sense_key":"journal%1:10:00::","synset_id":"journal.n.02"},{"index":3,"sense_key":"journal%1:21:00::","synset_id":"daybook.n.01"},{"index":4,"sense_key":"journal%1:06:01::","synset_id":"journal.n.04"},{"index":5,"sense_key":"journal%1:06:00::","synset_id":"journal.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"journal%1:10:00::","call_ids":["semeval2013.d007.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"journal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015769999999999998,"input_uncached_usd":0.0000233,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0669987499713898}
{"item_id":"semeval2013.d007.s006.t000","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001668,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0337921250029467}
{"item_id":"semeval2013.d007.s006.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005376500000000001,"input_uncached_usd":0.00004485,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.893052375002299}
{"item_id":"semeval2013.d007.s006.t002","gold_sense_keys":["carbon%1:27:00::"],"candidates":[{"index":1,"sense_key":"carbon%1:27:00::","synset_id":"carbon.n.01"},{"index":2,"sense_key":"carbon%1:27:01::","synset_id":"carbon_paper.n.01"},{"index":3,"sense_key":"carbon%1:06:00::","synset_id":"carbon.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"carbon%1:27:00::","call_ids":["semeval2013.d007.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"carbon%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015749999999999998,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1077826659893617}
{"item_id":"semeval2013.d007.s006.t006","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033549999999999997,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.229767124983482}
{"item_id":"semeval2013.d007.s007.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:01::","call_ids":["semeval2013.d007.s007.t003__v1__a1","semeval2013.d007.s007.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":969,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00057325,"input_uncached_usd":0.00004845,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.970236541994382}
{"item_id":"semeval2013.d007.s007.t004","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s007.t004__v1__a1","semeval2013.d007.s007.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":913,"cached_input_tokens":0,"output_tokens":736,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00034005,"input_uncached_usd":0.00004565,"input_cached_usd":0.0,"output_usd":0.0002944,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.583911166933831}
{"item_id":"semeval2013.d007.s008.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2013.d007.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006278000000000001,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.246420708019286}
{"item_id":"semeval2013.d007.s009.t000","gold_sense_keys":["find%1:09:00::"],"candidates":[{"index":1,"sense_key":"find%1:09:00::","synset_id":"discovery.n.03"},{"index":2,"sense_key":"find%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"find%1:09:00::","call_ids":["semeval2013.d007.s009.t000__v1__a1","semeval2013.d007.s009.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"find%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":819,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00064255,"input_uncached_usd":0.00004094999999999999,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.803358582954388}
{"item_id":"semeval2013.d007.s009.t001","gold_sense_keys":["search%1:04:00::","search%1:04:02::"],"candidates":[{"index":1,"sense_key":"search%1:04:00::","synset_id":"search.n.01"},{"index":2,"sense_key":"search%1:04:02::","synset_id":"search.n.02"},{"index":3,"sense_key":"search%1:22:00::","synset_id":"search.n.03"},{"index":4,"sense_key":"search%1:09:00::","synset_id":"search.n.04"},{"index":5,"sense_key":"search%1:04:01::","synset_id":"search.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"search%1:04:02::","call_ids":["semeval2013.d007.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"search%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044219999999999996,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.742025624960661}
{"item_id":"semeval2013.d007.s009.t002","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048659999999999996,"input_uncached_usd":0.000044999999999999996,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.272721749963239}
{"item_id":"semeval2013.d007.s009.t003","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034675,"input_uncached_usd":0.00003315,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.934337583021261}
{"item_id":"semeval2013.d007.s009.t005","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020494999999999997,"input_uncached_usd":0.000044949999999999995,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5227348750340752}
{"item_id":"semeval2013.d007.s009.t006","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":900,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.000589,"input_uncached_usd":0.000044999999999999996,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.432808832963929}
{"item_id":"semeval2013.d007.s010.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2013.d007.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017314999999999998,"input_uncached_usd":0.00003875,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.248141125019174}
{"item_id":"semeval2013.d007.s010.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037374999999999995,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0950551250134595}
{"item_id":"semeval2013.d007.s011.t002","gold_sense_keys":["detail%1:09:00::"],"candidates":[{"index":1,"sense_key":"detail%1:09:00::","synset_id":"detail.n.01"},{"index":2,"sense_key":"detail%1:24:00::","synset_id":"detail.n.02"},{"index":3,"sense_key":"detail%1:10:00::","synset_id":"detail.n.03"},{"index":4,"sense_key":"detail%1:14:01::","synset_id":"detail.n.04"},{"index":5,"sense_key":"detail%1:14:00::","synset_id":"contingent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"detail%1:24:00::","call_ids":["semeval2013.d007.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"detail%1:24:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036449999999999997,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.642774042033125}
{"item_id":"semeval2013.d007.s011.t003","gold_sense_keys":["find%1:09:00::"],"candidates":[{"index":1,"sense_key":"find%1:09:00::","synset_id":"discovery.n.03"},{"index":2,"sense_key":"find%1:04:00::","synset_id":"discovery.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%1:04:00::","call_ids":["semeval2013.d007.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005882999999999999,"input_uncached_usd":0.000018299999999999998,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.28331516700564}
{"item_id":"semeval2013.d007.s011.t004","gold_sense_keys":["speculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"speculation%1:10:03::","synset_id":"guess.n.01"},{"index":2,"sense_key":"speculation%1:09:00::","synset_id":"speculation.n.02"},{"index":3,"sense_key":"speculation%1:21:00::","synset_id":"speculation.n.03"},{"index":4,"sense_key":"speculation%1:09:01::","synset_id":"meditation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"speculation%1:09:00::","call_ids":["semeval2013.d007.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"speculation%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041665,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.142004749970511}
{"item_id":"semeval2013.d007.s011.t006","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2013.d007.s011.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003199,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.19018924998818}
{"item_id":"semeval2013.d007.s011.t007","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discovery%1:04:00::","call_ids":["semeval2013.d007.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discovery%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036135,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.316549958952237}
{"item_id":"semeval2013.d007.s013.t000","gold_sense_keys":["discovery%1:09:00::","discovery%1:10:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"discovery%1:10:00::","call_ids":["semeval2013.d007.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"discovery%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061865,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.793688541976735}
{"item_id":"semeval2013.d007.s013.t001","gold_sense_keys":["environment%1:26:00::"],"candidates":[{"index":1,"sense_key":"environment%1:26:00::","synset_id":"environment.n.01"},{"index":2,"sense_key":"environment%1:15:00::","synset_id":"environment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"environment%1:15:00::","call_ids":["semeval2013.d007.s013.t001__v1__a1","semeval2013.d007.s013.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"environment%1:15:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":887,"cached_input_tokens":0,"output_tokens":3040,"reasoning_output_tokens":3008},"cost":{"currency":"USD","total_usd":0.00126035,"input_uncached_usd":0.000044349999999999994,"input_cached_usd":0.0,"output_usd":0.001216,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.39062424999429}
{"item_id":"semeval2013.d007.s013.t002","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00064145,"input_uncached_usd":0.00002065,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.348723292001523}
{"item_id":"semeval2013.d007.s013.t003","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019329999999999998,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.44918575003976}
{"item_id":"semeval2013.d007.s013.t005","gold_sense_keys":["set%1:14:00::"],"candidates":[{"index":1,"sense_key":"set%1:14:00::","synset_id":"set.n.01"},{"index":2,"sense_key":"set%1:14:02::","synset_id":"set.n.02"},{"index":3,"sense_key":"set%1:04:00::","synset_id":"set.n.03"},{"index":4,"sense_key":"set%1:06:00::","synset_id":"stage_set.n.01"},{"index":5,"sense_key":"set%1:14:01::","synset_id":"set.n.05"},{"index":6,"sense_key":"set%1:09:01::","synset_id":"bent.n.01"},{"index":7,"sense_key":"set%1:04:01::","synset_id":"set.n.07"},{"index":8,"sense_key":"set%1:28:00::","synset_id":"set.n.08"},{"index":9,"sense_key":"set%1:22:00::","synset_id":"hardening.n.02"},{"index":10,"sense_key":"set%1:18:00::","synset_id":"set.n.10"},{"index":11,"sense_key":"set%1:11:00::","synset_id":"set.n.11"},{"index":12,"sense_key":"set%1:09:00::","synset_id":"set.n.12"},{"index":13,"sense_key":"set%1:06:01::","synset_id":"set.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%1:14:00::","call_ids":["semeval2013.d007.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040964999999999996,"input_uncached_usd":0.00004485,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.900989124958869}
{"item_id":"semeval2013.d007.s013.t006","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":658,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011609999999999999,"input_uncached_usd":0.0000329,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.48095941694919}
{"item_id":"semeval2013.d007.s013.t007","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":907,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025655,"input_uncached_usd":0.00004535,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.421156666998286}
{"item_id":"semeval2013.d007.s013.t009","gold_sense_keys":["space%1:15:01::"],"candidates":[{"index":1,"sense_key":"space%1:03:00::","synset_id":"space.n.01"},{"index":2,"sense_key":"space%1:25:00::","synset_id":"space.n.02"},{"index":3,"sense_key":"space%1:15:00::","synset_id":"space.n.03"},{"index":4,"sense_key":"space%1:15:01::","synset_id":"outer_space.n.01"},{"index":5,"sense_key":"space%1:10:00::","synset_id":"space.n.05"},{"index":6,"sense_key":"space%1:28:00::","synset_id":"distance.n.05"},{"index":7,"sense_key":"space%1:10:01::","synset_id":"space.n.07"},{"index":8,"sense_key":"space%1:10:02::","synset_id":"space.n.08"},{"index":9,"sense_key":"space%1:06:00::","synset_id":"quad.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"space%1:15:01::","call_ids":["semeval2013.d007.s013.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"space%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00063265,"input_uncached_usd":0.000037449999999999995,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.876616917026695}
{"item_id":"semeval2013.d007.s013.t010","gold_sense_keys":["science%1:09:00::"],"candidates":[{"index":1,"sense_key":"science%1:09:00::","synset_id":"science.n.01"},{"index":2,"sense_key":"science%1:09:02::","synset_id":"skill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"science%1:09:00::","call_ids":["semeval2013.d007.s013.t010__v1__a1","semeval2013.d007.s013.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"science%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007229500000000001,"input_uncached_usd":0.00004455,"input_cached_usd":0.0,"output_usd":0.0006783999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.767845041991677}
{"item_id":"semeval2013.d007.s014.t000","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"part%1:09:02::","call_ids":["semeval2013.d007.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"part%1:09:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":932,"cached_input_tokens":0,"output_tokens":3088,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.0012817999999999998,"input_uncached_usd":0.0000466,"input_cached_usd":0.0,"output_usd":0.0012351999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.3616396660218}
{"item_id":"semeval2013.d007.s014.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d007.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020529999999999998,"input_uncached_usd":0.000019699999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9196025409619324}
{"item_id":"semeval2013.d007.s015.t000","gold_sense_keys":["work%1:06:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"work%1:06:00::","call_ids":["semeval2013.d007.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"work%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":674,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008593,"input_uncached_usd":0.0000337,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.56615225004498}
{"item_id":"semeval2013.d007.s015.t001","gold_sense_keys":["substitution%1:11:00::"],"candidates":[{"index":1,"sense_key":"substitution%1:11:00::","synset_id":"substitution.n.01"},{"index":2,"sense_key":"substitution%1:04:00::","synset_id":"substitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"substitution%1:04:00::","call_ids":["semeval2013.d007.s015.t001__v1__a1","semeval2013.d007.s015.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"substitution%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00059505,"input_uncached_usd":0.00004465,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.40337500005262}
{"item_id":"semeval2013.d007.s015.t002","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021719999999999997,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.249342624971177}
{"item_id":"semeval2013.d007.s015.t004","gold_sense_keys":["structure%1:07:00::"],"candidates":[{"index":1,"sense_key":"structure%1:06:00::","synset_id":"structure.n.01"},{"index":2,"sense_key":"structure%1:07:00::","synset_id":"structure.n.02"},{"index":3,"sense_key":"structure%1:09:00::","synset_id":"structure.n.03"},{"index":4,"sense_key":"structure%1:08:00::","synset_id":"structure.n.04"},{"index":5,"sense_key":"structure%1:14:00::","synset_id":"social_organization.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"structure%1:09:00::","call_ids":["semeval2013.d007.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"structure%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":1681,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007010999999999999,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0006724,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.08218124997802}
{"item_id":"semeval2013.d007.s016.t000","gold_sense_keys":["discovery%1:04:00::"],"candidates":[{"index":1,"sense_key":"discovery%1:04:00::","synset_id":"discovery.n.01"},{"index":2,"sense_key":"discovery%1:10:00::","synset_id":"discovery.n.02"},{"index":3,"sense_key":"discovery%1:09:00::","synset_id":"discovery.n.03"},{"index":4,"sense_key":"discovery%1:10:01::","synset_id":"discovery.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"discovery%1:09:00::","call_ids":["semeval2013.d007.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"discovery%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007462,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.997403874993324}
{"item_id":"semeval2013.d007.s017.t003","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2013.d007.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055515,"input_uncached_usd":0.00003675,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.407304666994605}
{"item_id":"semeval2013.d007.s017.t004","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003109,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.057784457982052}
{"item_id":"semeval2013.d007.s017.t005","gold_sense_keys":["level%1:07:00::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"level%1:07:00::","call_ids":["semeval2013.d007.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"level%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019414999999999998,"input_uncached_usd":0.000034149999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5663748750230297}
{"item_id":"semeval2013.d007.s018.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s018.t000__v1__a1","semeval2013.d007.s018.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":997,"cached_input_tokens":0,"output_tokens":1888,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0008050500000000001,"input_uncached_usd":0.00004984999999999999,"input_cached_usd":0.0,"output_usd":0.0007552,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.460580749961082}
{"item_id":"semeval2013.d007.s018.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013115,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9613580419681966}
{"item_id":"semeval2013.d007.s019.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030940000000000004,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.95290129102068}
{"item_id":"semeval2013.d007.s019.t002","gold_sense_keys":["behavior%1:26:00::"],"candidates":[{"index":1,"sense_key":"behavior%1:04:00::","synset_id":"behavior.n.01"},{"index":2,"sense_key":"behavior%1:26:00::","synset_id":"behavior.n.02"},{"index":3,"sense_key":"behavior%1:07:00::","synset_id":"demeanor.n.01"},{"index":4,"sense_key":"behavior%1:04:01::","synset_id":"behavior.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"behavior%1:26:00::","call_ids":["semeval2013.d007.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"behavior%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013145,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4041039580479264}
{"item_id":"semeval2013.d007.s019.t003","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051245,"input_uncached_usd":0.00001965,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.719112666964065}
{"item_id":"semeval2013.d007.s020.t000","gold_sense_keys":["result%1:19:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["semeval2013.d007.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026274999999999996,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.313547917001415}
{"item_id":"semeval2013.d007.s020.t001","gold_sense_keys":["organism%1:03:00::"],"candidates":[{"index":1,"sense_key":"organism%1:03:00::","synset_id":"organism.n.01"},{"index":2,"sense_key":"organism%1:14:00::","synset_id":"organism.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"organism%1:03:00::","call_ids":["semeval2013.d007.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"organism%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004096,"input_uncached_usd":0.0000192,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.9672520000021905}
{"item_id":"semeval2013.d007.s020.t002","gold_sense_keys":["hard_time%1:26:00::"],"candidates":[{"index":1,"sense_key":"hard_time%1:26:00::","synset_id":"hard_time.n.01"},{"index":2,"sense_key":"hard_time%1:28:00::","synset_id":"hard_time.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard_time%1:26:00::","call_ids":["semeval2013.d007.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard_time%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012815,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8955614579608664}
{"item_id":"semeval2013.d007.s020.t003","gold_sense_keys":["element%1:27:00::"],"candidates":[{"index":1,"sense_key":"element%1:09:00::","synset_id":"component.n.01"},{"index":2,"sense_key":"element%1:06:00::","synset_id":"component.n.03"},{"index":3,"sense_key":"element%1:27:00::","synset_id":"chemical_element.n.01"},{"index":4,"sense_key":"element%1:15:00::","synset_id":"element.n.04"},{"index":5,"sense_key":"element%1:27:01::","synset_id":"element.n.05"},{"index":6,"sense_key":"element%1:26:00::","synset_id":"element.n.06"},{"index":7,"sense_key":"element%1:25:00::","synset_id":"element.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"element%1:27:00::","call_ids":["semeval2013.d007.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"element%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011465,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.163800165988505}
{"item_id":"semeval2013.d007.s021.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:01::","call_ids":["semeval2013.d007.s021.t000__v1__a1","semeval2013.d007.s021.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00067475,"input_uncached_usd":0.000047550000000000004,"input_cached_usd":0.0,"output_usd":0.0006272,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.901567708991934}
{"item_id":"semeval2013.d007.s021.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038599999999999995,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.6414177500410005}
{"item_id":"semeval2013.d007.s022.t002","gold_sense_keys":["concentration%1:07:02::"],"candidates":[{"index":1,"sense_key":"concentration%1:07:02::","synset_id":"concentration.n.01"},{"index":2,"sense_key":"concentration%1:07:03::","synset_id":"concentration.n.02"},{"index":3,"sense_key":"concentration%1:04:01::","synset_id":"concentration.n.03"},{"index":4,"sense_key":"concentration%1:11:00::","synset_id":"concentration.n.04"},{"index":5,"sense_key":"concentration%1:09:00::","synset_id":"concentration.n.05"},{"index":6,"sense_key":"concentration%1:04:02::","synset_id":"concentration.n.06"},{"index":7,"sense_key":"concentration%1:07:01::","synset_id":"assiduity.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"concentration%1:07:02::","call_ids":["semeval2013.d007.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"concentration%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001636,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.387553999957163}
{"item_id":"semeval2013.d007.s022.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038954999999999995,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.228221166995354}
{"item_id":"semeval2013.d007.s023.t001","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023534999999999997,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.027334375015926}
{"item_id":"semeval2013.d007.s023.t002","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t002__v1__a1","semeval2013.d007.s023.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":3360,"reasoning_output_tokens":3328},"cost":{"currency":"USD","total_usd":0.00139745,"input_uncached_usd":0.00005345,"input_cached_usd":0.0,"output_usd":0.001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":22.0166947079706}
{"item_id":"semeval2013.d007.s023.t003","gold_sense_keys":["possibility%1:26:00::"],"candidates":[{"index":1,"sense_key":"possibility%1:09:03::","synset_id":"possibility.n.01"},{"index":2,"sense_key":"possibility%1:26:00::","synset_id":"possibility.n.02"},{"index":3,"sense_key":"possibility%1:09:00::","synset_id":"hypothesis.n.02"},{"index":4,"sense_key":"possibility%1:09:01::","synset_id":"possibility.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"possibility%1:26:00::","call_ids":["semeval2013.d007.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"possibility%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":567,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041875,"input_uncached_usd":0.000028349999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.687656291993335}
{"item_id":"semeval2013.d007.s023.t005","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s023.t005__v1__a1","semeval2013.d007.s023.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":2144,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.0009110499999999998,"input_uncached_usd":0.00005345,"input_cached_usd":0.0,"output_usd":0.0008576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.504006707982626}
{"item_id":"semeval2013.d007.s023.t006","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002465,"input_uncached_usd":0.0000353,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7868335830280557}
{"item_id":"semeval2013.d007.s023.t007","gold_sense_keys":["planet%1:17:00::","planet%1:17:01::"],"candidates":[{"index":1,"sense_key":"planet%1:17:00::","synset_id":"planet.n.01"},{"index":2,"sense_key":"planet%1:18:00::","synset_id":"satellite.n.02"},{"index":3,"sense_key":"planet%1:17:01::","synset_id":"planet.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"planet%1:17:01::","call_ids":["semeval2013.d007.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"planet%1:17:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031445,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.580501708027441}
{"item_id":"semeval2013.d007.s023.t008","gold_sense_keys":["moon%1:17:00::"],"candidates":[{"index":1,"sense_key":"moon%1:17:01::","synset_id":"moon.n.01"},{"index":2,"sense_key":"moon%1:17:02::","synset_id":"moon.n.02"},{"index":3,"sense_key":"moon%1:28:00::","synset_id":"lunar_month.n.01"},{"index":4,"sense_key":"moon%1:19:00::","synset_id":"moonlight.n.01"},{"index":5,"sense_key":"moon%1:18:00::","synset_id":"moon.n.05"},{"index":6,"sense_key":"moon%1:17:00::","synset_id":"moon.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"moon%1:17:00::","call_ids":["semeval2013.d007.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"moon%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034629999999999996,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.474063083005603}
{"item_id":"semeval2013.d007.s023.t009","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"arsenic%1:27:01::","call_ids":["semeval2013.d007.s023.t009__v1__a1","semeval2013.d007.s023.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"arsenic%1:27:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1069,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00060385,"input_uncached_usd":0.00005345,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.068950916989706}
{"item_id":"semeval2013.d007.s024.t002","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d007.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00060055,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.00506687501911}
{"item_id":"semeval2013.d007.s024.t003","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s024.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044195,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.584805292019155}
{"item_id":"semeval2013.d007.s024.t004","gold_sense_keys":["phosphorus%1:27:00::"],"candidates":[{"index":1,"sense_key":"phosphorus%1:27:00::","synset_id":"phosphorus.n.01"},{"index":2,"sense_key":"phosphorus%1:17:00::","synset_id":"morning_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"phosphorus%1:27:00::","call_ids":["semeval2013.d007.s024.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"phosphorus%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020979999999999998,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5323911249870434}
{"item_id":"semeval2013.d007.s024.t005","gold_sense_keys":["example%1:11:00::"],"candidates":[{"index":1,"sense_key":"example%1:09:00::","synset_id":"example.n.01"},{"index":2,"sense_key":"example%1:09:02::","synset_id":"model.n.07"},{"index":3,"sense_key":"example%1:09:01::","synset_id":"exemplar.n.01"},{"index":4,"sense_key":"example%1:10:00::","synset_id":"example.n.04"},{"index":5,"sense_key":"example%1:11:00::","synset_id":"case.n.01"},{"index":6,"sense_key":"example%1:04:00::","synset_id":"exercise.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"example%1:11:00::","call_ids":["semeval2013.d007.s024.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"example%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":650,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003205,"input_uncached_usd":0.0000325,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.926869165967219}
{"item_id":"semeval2013.d007.s025.t000","gold_sense_keys":["arsenic%1:27:00::"],"candidates":[{"index":1,"sense_key":"arsenic%1:27:01::","synset_id":"arsenic.n.01"},{"index":2,"sense_key":"arsenic%1:27:00::","synset_id":"arsenic.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"arsenic%1:27:00::","call_ids":["semeval2013.d007.s025.t000__v1__a1","semeval2013.d007.s025.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"arsenic%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1109,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007594500000000001,"input_uncached_usd":0.00005544999999999999,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.264087457966525}
{"item_id":"semeval2013.d007.s025.t001","gold_sense_keys":["times%1:28:01::"],"candidates":[{"index":1,"sense_key":"times%1:28:01::","synset_id":"times.n.01"},{"index":2,"sense_key":"times%1:04:00::","synset_id":"multiplication.n.03"},{"index":3,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":4,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":7,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"clock_time%1:28:00::","synset_id":"clock_time.n.01"},{"index":10,"sense_key":"fourth_dimension%1:28:00::","synset_id":"fourth_dimension.n.01"},{"index":11,"sense_key":"meter%1:07:00::","synset_id":"meter.n.04"},{"index":12,"sense_key":"prison_term%1:28:00::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"times%1:28:01::","call_ids":["semeval2013.d007.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"times%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":926,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030869999999999997,"input_uncached_usd":0.0000463,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.207334084028844}
{"item_id":"semeval2013.d007.s025.t002","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":724,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017059999999999997,"input_uncached_usd":0.0000362,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.163725833001081}
{"item_id":"semeval2013.d007.s026.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d007.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":868,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005618,"input_uncached_usd":0.0000434,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.298497042036615}
{"item_id":"semeval2013.d007.s026.t001","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s026.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":964,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007714,"input_uncached_usd":0.0000482,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.823358374997042}
{"item_id":"semeval2013.d007.s026.t002","gold_sense_keys":["life%1:19:00::"],"candidates":[{"index":1,"sense_key":"life%1:26:01::","synset_id":"life.n.01"},{"index":2,"sense_key":"life%1:09:00::","synset_id":"life.n.02"},{"index":3,"sense_key":"life%1:26:02::","synset_id":"life.n.03"},{"index":4,"sense_key":"life%1:26:00::","synset_id":"animation.n.01"},{"index":5,"sense_key":"life%1:28:00::","synset_id":"life.n.05"},{"index":6,"sense_key":"life%1:28:01::","synset_id":"life.n.06"},{"index":7,"sense_key":"life%1:28:02::","synset_id":"life.n.07"},{"index":8,"sense_key":"life%1:18:00::","synset_id":"life.n.08"},{"index":9,"sense_key":"life%1:07:01::","synset_id":"liveliness.n.02"},{"index":10,"sense_key":"life%1:03:00::","synset_id":"life.n.10"},{"index":11,"sense_key":"life%1:19:00::","synset_id":"life.n.11"},{"index":12,"sense_key":"life%1:10:00::","synset_id":"biography.n.01"},{"index":13,"sense_key":"life%1:16:00::","synset_id":"life.n.13"},{"index":14,"sense_key":"life%1:28:03::","synset_id":"life_sentence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"life%1:19:00::","call_ids":["semeval2013.d007.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"life%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":965,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043865,"input_uncached_usd":0.00004825,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.358528749959078}
{"item_id":"semeval2013.d007.s026.t003","gold_sense_keys":["earth%1:17:00::"],"candidates":[{"index":1,"sense_key":"earth%1:17:00::","synset_id":"earth.n.01"},{"index":2,"sense_key":"earth%1:27:00::","synset_id":"earth.n.02"},{"index":3,"sense_key":"earth%1:17:01::","synset_id":"land.n.04"},{"index":4,"sense_key":"earth%1:15:00::","synset_id":"earth.n.04"},{"index":5,"sense_key":"earth%1:27:01::","synset_id":"earth.n.05"},{"index":6,"sense_key":"earth%1:09:00::","synset_id":"worldly_concern.n.01"},{"index":7,"sense_key":"earth%1:06:00::","synset_id":"ground.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"earth%1:17:00::","call_ids":["semeval2013.d007.s026.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"earth%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027319999999999997,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.363213791046292}
{"item_id":"semeval2013.d007.s026.t004","gold_sense_keys":["part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2013.d007.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":990,"cached_input_tokens":0,"output_tokens":3281,"reasoning_output_tokens":3264},"cost":{"currency":"USD","total_usd":0.0013618999999999999,"input_uncached_usd":0.0000495,"input_cached_usd":0.0,"output_usd":0.0013124,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":21.40697700000601}
{"item_id":"semeval2013.d007.s026.t006","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d007.s026.t006__v1__a1","semeval2013.d007.s026.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":951,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041874999999999996,"input_uncached_usd":0.000047550000000000004,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.77828175103059}
{"item_id":"semeval2013.d008.s000.t000","gold_sense_keys":["routine%1:04:00::"],"candidates":[{"index":1,"sense_key":"routine%1:04:00::","synset_id":"routine.n.01"},{"index":2,"sense_key":"routine%1:10:01::","synset_id":"act.n.04"},{"index":3,"sense_key":"routine%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"routine%1:04:00::","call_ids":["semeval2013.d008.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"routine%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":294,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003027,"input_uncached_usd":0.0000147,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.995843999960925}
{"item_id":"semeval2013.d008.s001.t000","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00070655,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.071315540990327}
{"item_id":"semeval2013.d008.s001.t003","gold_sense_keys":["backing%1:04:00::"],"candidates":[{"index":1,"sense_key":"backing%1:04:00::","synset_id":"backing.n.01"},{"index":2,"sense_key":"backing%1:06:00::","synset_id":"backing.n.02"},{"index":3,"sense_key":"backing%1:21:00::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"backing%1:04:00::","call_ids":["semeval2013.d008.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"backing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":308,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003034,"input_uncached_usd":0.000015399999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.349815582972951}
{"item_id":"semeval2013.d008.s002.t000","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":284,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022539999999999998,"input_uncached_usd":0.0000142,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.028297749988269}
{"item_id":"semeval2013.d008.s002.t001","gold_sense_keys":["worry%1:09:00::"],"candidates":[{"index":1,"sense_key":"worry%1:09:00::","synset_id":"concern.n.04"},{"index":2,"sense_key":"worry%1:12:00::","synset_id":"worry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worry%1:09:00::","call_ids":["semeval2013.d008.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worry%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":295,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00068675,"input_uncached_usd":0.00001475,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.860240790992975}
{"item_id":"semeval2013.d008.s003.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":312,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025239999999999996,"input_uncached_usd":0.0000156,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.17787362501258}
{"item_id":"semeval2013.d008.s003.t001","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015155,"input_uncached_usd":0.00001715,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0680328750167973}
{"item_id":"semeval2013.d008.s004.t000","gold_sense_keys":["finding%1:17:00::"],"candidates":[{"index":1,"sense_key":"finding%1:04:00::","synset_id":"determination.n.01"},{"index":2,"sense_key":"finding%1:04:01::","synset_id":"finding.n.02"},{"index":3,"sense_key":"finding%1:17:00::","synset_id":"finding.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"finding%1:17:00::","call_ids":["semeval2013.d008.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"finding%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020355,"input_uncached_usd":0.00001795,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.087909334048163}
{"item_id":"semeval2013.d008.s004.t001","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015384999999999997,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4843407080043107}
{"item_id":"semeval2013.d008.s004.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003381,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.34835316601675}
{"item_id":"semeval2013.d008.s005.t000","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010364999999999999,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2516846249927767}
{"item_id":"semeval2013.d008.s005.t001","gold_sense_keys":["change%1:19:00::","change%1:24:00::"],"candidates":[{"index":1,"sense_key":"change%1:11:00::","synset_id":"change.n.01"},{"index":2,"sense_key":"change%1:24:00::","synset_id":"change.n.02"},{"index":3,"sense_key":"change%1:04:00::","synset_id":"change.n.03"},{"index":4,"sense_key":"change%1:19:00::","synset_id":"change.n.04"},{"index":5,"sense_key":"change%1:21:02::","synset_id":"change.n.05"},{"index":6,"sense_key":"change%1:06:01::","synset_id":"change.n.06"},{"index":7,"sense_key":"change%1:06:00::","synset_id":"change.n.07"},{"index":8,"sense_key":"change%1:21:01::","synset_id":"change.n.08"},{"index":9,"sense_key":"change%1:21:03::","synset_id":"change.n.09"},{"index":10,"sense_key":"change%1:07:00::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"change%1:19:00::","call_ids":["semeval2013.d008.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"change%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":2641,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.0010907999999999998,"input_uncached_usd":0.000034399999999999996,"input_cached_usd":0.0,"output_usd":0.0010563999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.374630625010468}
{"item_id":"semeval2013.d008.s005.t002","gold_sense_keys":["attitude%1:09:00::"],"candidates":[{"index":1,"sense_key":"attitude%1:09:00::","synset_id":"attitude.n.01"},{"index":2,"sense_key":"attitude%1:07:01::","synset_id":"position.n.04"},{"index":3,"sense_key":"attitude%1:04:00::","synset_id":"attitude.n.03"},{"index":4,"sense_key":"attitude%1:24:00::","synset_id":"attitude.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attitude%1:09:00::","call_ids":["semeval2013.d008.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attitude%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012989999999999999,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6651554169948213}
{"item_id":"semeval2013.d008.s006.t000","gold_sense_keys":["thing%1:09:02::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"thing%1:03:00::","call_ids":["semeval2013.d008.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"thing%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005516000000000001,"input_uncached_usd":0.0000332,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.958067708008457}
{"item_id":"semeval2013.d008.s006.t001","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2013.d008.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015825,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.350602791993879}
{"item_id":"semeval2013.d008.s006.t002","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020879999999999998,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.925190042005852}
{"item_id":"semeval2013.d008.s006.t003","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003087,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.462044916988816}
{"item_id":"semeval2013.d008.s007.t001","gold_sense_keys":["mood%1:12:00::","mood%1:26:00::"],"candidates":[{"index":1,"sense_key":"mood%1:12:00::","synset_id":"temper.n.02"},{"index":2,"sense_key":"mood%1:26:00::","synset_id":"climate.n.02"},{"index":3,"sense_key":"mood%1:24:00::","synset_id":"mood.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mood%1:26:00::","call_ids":["semeval2013.d008.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mood%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023145,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.611549582972657}
{"item_id":"semeval2013.d008.s008.t000","gold_sense_keys":["second%5:00:00:ordinal:00"],"candidates":[{"index":1,"sense_key":"second%5:00:00:ordinal:00","synset_id":"second.s.01"},{"index":2,"sense_key":"second%3:00:00::","synset_id":"second.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"second%5:00:00:ordinal:00","call_ids":["semeval2013.d008.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"second%5:00:00:ordinal:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":365,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00022945,"input_uncached_usd":0.00001825,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.541664040996693}
{"item_id":"semeval2013.d008.s008.t001","gold_sense_keys":["stability%1:07:01::"],"candidates":[{"index":1,"sense_key":"stability%1:07:00::","synset_id":"stability.n.01"},{"index":2,"sense_key":"stability%1:26:00::","synset_id":"stability.n.02"},{"index":3,"sense_key":"stability%1:07:01::","synset_id":"constancy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"stability%1:07:01::","call_ids":["semeval2013.d008.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"stability%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035874999999999997,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.000939374964219}
{"item_id":"semeval2013.d008.s008.t002","gold_sense_keys":["attitude%1:09:00::"],"candidates":[{"index":1,"sense_key":"attitude%1:09:00::","synset_id":"attitude.n.01"},{"index":2,"sense_key":"attitude%1:07:01::","synset_id":"position.n.04"},{"index":3,"sense_key":"attitude%1:04:00::","synset_id":"attitude.n.03"},{"index":4,"sense_key":"attitude%1:24:00::","synset_id":"attitude.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attitude%1:09:00::","call_ids":["semeval2013.d008.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attitude%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001051,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.7560022919788025}
{"item_id":"semeval2013.d008.s008.t004","gold_sense_keys":["institution%1:09:00::"],"candidates":[{"index":1,"sense_key":"institution%1:14:00::","synset_id":"institution.n.01"},{"index":2,"sense_key":"institution%1:06:00::","synset_id":"institution.n.02"},{"index":3,"sense_key":"institution%1:09:00::","synset_id":"institution.n.03"},{"index":4,"sense_key":"institution%1:04:00::","synset_id":"initiation.n.02"},{"index":5,"sense_key":"institution%1:06:01::","synset_id":"mental_hospital.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"institution%1:14:00::","call_ids":["semeval2013.d008.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"institution%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026185,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.258156124968082}
{"item_id":"semeval2013.d008.s009.t000","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":816,"cached_input_tokens":0,"output_tokens":2577,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010715999999999998,"input_uncached_usd":0.000040799999999999996,"input_cached_usd":0.0,"output_usd":0.0010308,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.31711683300091}
{"item_id":"semeval2013.d008.s009.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003412,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.64801741699921}
{"item_id":"semeval2013.d008.s009.t003","gold_sense_keys":["south_america%1:17:00::"],"candidates":[{"index":1,"sense_key":"south_america%1:17:00::","synset_id":"south_america.n.01"},{"index":2,"sense_key":"south_america%1:14:00::","synset_id":"south_america.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"south_america%1:17:00::","call_ids":["semeval2013.d008.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"south_america%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035834999999999996,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.559951499977615}
{"item_id":"semeval2013.d008.s009.t004","gold_sense_keys":["table%1:14:00::"],"candidates":[{"index":1,"sense_key":"table%1:14:00::","synset_id":"table.n.01"},{"index":2,"sense_key":"table%1:06:01::","synset_id":"table.n.02"},{"index":3,"sense_key":"table%1:06:02::","synset_id":"table.n.03"},{"index":4,"sense_key":"table%1:17:00::","synset_id":"mesa.n.01"},{"index":5,"sense_key":"table%1:14:01::","synset_id":"table.n.05"},{"index":6,"sense_key":"table%1:13:01::","synset_id":"board.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"table%1:14:00::","call_ids":["semeval2013.d008.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"table%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003412,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.852315625001211}
{"item_id":"semeval2013.d008.s010.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021035,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.688814666995313}
{"item_id":"semeval2013.d008.s010.t004","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":812,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003542,"input_uncached_usd":0.0000406,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.910242832964286}
{"item_id":"semeval2013.d008.s010.t006","gold_sense_keys":["low%1:07:00::"],"candidates":[{"index":1,"sense_key":"low%1:26:00::","synset_id":"low.n.01"},{"index":2,"sense_key":"low%1:18:00::","synset_id":"low.n.02"},{"index":3,"sense_key":"low%1:07:00::","synset_id":"low.n.03"},{"index":4,"sense_key":"low%1:06:00::","synset_id":"first_gear.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"low%1:07:00::","call_ids":["semeval2013.d008.s010.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"low%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010724999999999999,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4737175420159474}
{"item_id":"semeval2013.d008.s010.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":494,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004151,"input_uncached_usd":0.000024699999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.023478459042963}
{"item_id":"semeval2013.d008.s011.t001","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d008.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018375,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5869212090037763}
{"item_id":"semeval2013.d008.s011.t002","gold_sense_keys":["drop%1:07:00::"],"candidates":[{"index":1,"sense_key":"drop%1:25:00::","synset_id":"drop.n.01"},{"index":2,"sense_key":"drop%1:23:00::","synset_id":"drop.n.02"},{"index":3,"sense_key":"drop%1:07:00::","synset_id":"drop.n.03"},{"index":4,"sense_key":"drop%1:17:00::","synset_id":"cliff.n.01"},{"index":5,"sense_key":"drop%1:15:00::","synset_id":"drop.n.05"},{"index":6,"sense_key":"drop%1:11:00::","synset_id":"drop.n.06"},{"index":7,"sense_key":"drop%1:06:01::","synset_id":"drop_curtain.n.01"},{"index":8,"sense_key":"drop%1:06:00::","synset_id":"drop.n.08"},{"index":9,"sense_key":"drop%1:04:00::","synset_id":"drop.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"drop%1:07:00::","call_ids":["semeval2013.d008.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"drop%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":712,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011879999999999999,"input_uncached_usd":0.0000356,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.076677125005517}
{"item_id":"semeval2013.d008.s011.t003","gold_sense_keys":["output%1:04:00::","output%1:06:01::","output%1:23:00::"],"candidates":[{"index":1,"sense_key":"output%1:06:00::","synset_id":"end_product.n.01"},{"index":2,"sense_key":"output%1:04:00::","synset_id":"output.n.02"},{"index":3,"sense_key":"output%1:10:02::","synset_id":"output_signal.n.01"},{"index":4,"sense_key":"output%1:23:00::","synset_id":"output.n.04"},{"index":5,"sense_key":"output%1:06:01::","synset_id":"output.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"output%1:23:00::","call_ids":["semeval2013.d008.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"output%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023539999999999998,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.862138791999314}
{"item_id":"semeval2013.d008.s011.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002879,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.331857750017662}
{"item_id":"semeval2013.d008.s012.t000","gold_sense_keys":["half%1:23:00::"],"candidates":[{"index":1,"sense_key":"half%1:23:00::","synset_id":"one-half.n.01"},{"index":2,"sense_key":"half%1:28:00::","synset_id":"half.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"half%1:23:00::","call_ids":["semeval2013.d008.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"half%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012974999999999998,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5085068750195205}
{"item_id":"semeval2013.d008.s012.t001","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041085,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.728132082964294}
{"item_id":"semeval2013.d008.s012.t003","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015595,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0612633749842644}
{"item_id":"semeval2013.d008.s012.t005","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047099999999999996,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.801567292015534}
{"item_id":"semeval2013.d008.s012.t006","gold_sense_keys":["democrat%1:18:00::"],"candidates":[{"index":1,"sense_key":"democrat%1:18:01::","synset_id":"democrat.n.01"},{"index":2,"sense_key":"democrat%1:18:00::","synset_id":"democrat.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democrat%1:18:00::","call_ids":["semeval2013.d008.s012.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democrat%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038439999999999997,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.113686292024795}
{"item_id":"semeval2013.d008.s012.t007","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:00::","synset_id":"average.n.01"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:01::","synset_id":"average.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2013.d008.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023354999999999998,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.855628332996275}
{"item_id":"semeval2013.d008.s012.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s012.t008__v1__a1","semeval2013.d008.s012.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056585,"input_uncached_usd":0.000041049999999999995,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.482845749997068}
{"item_id":"semeval2013.d008.s013.t000","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s013.t000__v1__a1","semeval2013.d008.s013.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":935,"cached_input_tokens":0,"output_tokens":1505,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00064875,"input_uncached_usd":0.00004675,"input_cached_usd":0.0,"output_usd":0.0006019999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.534985041012987}
{"item_id":"semeval2013.d008.s013.t002","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031925,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.619578333979007}
{"item_id":"semeval2013.d008.s013.t003","gold_sense_keys":["democracy%1:14:00::"],"candidates":[{"index":1,"sense_key":"democracy%1:09:00::","synset_id":"democracy.n.01"},{"index":2,"sense_key":"democracy%1:14:00::","synset_id":"democracy.n.02"},{"index":3,"sense_key":"democracy%1:09:01::","synset_id":"majority_rule.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"democracy%1:14:00::","call_ids":["semeval2013.d008.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"democracy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031285,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.907940125034656}
{"item_id":"semeval2013.d008.s013.t005","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":2,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":4,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":5,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":6,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":7,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"},{"index":8,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":9,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":10,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":11,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":12,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":13,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"figure%1:07:01::","call_ids":["semeval2013.d008.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"figure%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":905,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005636500000000001,"input_uncached_usd":0.000045249999999999995,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.263641207944602}
{"item_id":"semeval2013.d008.s013.t006","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s013.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005213500000000001,"input_uncached_usd":0.00002855,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.8436185420141555}
{"item_id":"semeval2013.d008.s013.t007","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":4,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":5,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d008.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":2833,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.00116245,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.0011332,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.978306583012454}
{"item_id":"semeval2013.d008.s013.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053945,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.697225374984555}
{"item_id":"semeval2013.d008.s014.t000","gold_sense_keys":["confidence%1:12:00::","confidence%1:26:02::"],"candidates":[{"index":1,"sense_key":"confidence%1:09:00::","synset_id":"assurance.n.01"},{"index":2,"sense_key":"confidence%1:12:00::","synset_id":"confidence.n.02"},{"index":3,"sense_key":"confidence%1:26:02::","synset_id":"confidence.n.03"},{"index":4,"sense_key":"confidence%1:26:03::","synset_id":"confidence.n.04"},{"index":5,"sense_key":"confidence%1:10:00::","synset_id":"confidence.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"confidence%1:12:00::","call_ids":["semeval2013.d008.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"confidence%1:12:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044605,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.899662917014211}
{"item_id":"semeval2013.d008.s014.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":624,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004476,"input_uncached_usd":0.0000312,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.578054040961433}
{"item_id":"semeval2013.d008.s014.t002","gold_sense_keys":["congress%1:14:02::"],"candidates":[{"index":1,"sense_key":"congress%1:14:01::","synset_id":"congress.n.01"},{"index":2,"sense_key":"congress%1:14:00::","synset_id":"congress.n.02"},{"index":3,"sense_key":"congress%1:14:02::","synset_id":"congress.n.03"},{"index":4,"sense_key":"congress%1:04:01::","synset_id":"sexual_intercourse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"congress%1:14:02::","call_ids":["semeval2013.d008.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"congress%1:14:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031405,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.287829625012819}
{"item_id":"semeval2013.d008.s014.t003","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2013.d008.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008021,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.239518542017322}
{"item_id":"semeval2013.d008.s014.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029089999999999997,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.082840375020169}
{"item_id":"semeval2013.d008.s014.t005","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case%1:11:00::","call_ids":["semeval2013.d008.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1236,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002986,"input_uncached_usd":0.0000618,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.639056624961086}
{"item_id":"semeval2013.d008.s015.t000","gold_sense_keys":["decade%1:28:00::"],"candidates":[{"index":1,"sense_key":"decade%1:28:00::","synset_id":"decade.n.01"},{"index":2,"sense_key":"decade%1:23:00::","synset_id":"ten.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"decade%1:28:00::","call_ids":["semeval2013.d008.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"decade%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018309999999999998,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4332322499831207}
{"item_id":"semeval2013.d008.s015.t001","gold_sense_keys":["woe%1:26:00::"],"candidates":[{"index":1,"sense_key":"woe%1:26:00::","synset_id":"suffering.n.02"},{"index":2,"sense_key":"woe%1:12:00::","synset_id":"woe.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"woe%1:26:00::","call_ids":["semeval2013.d008.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"woe%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028435,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.851375875005033}
{"item_id":"semeval2013.d008.s015.t003","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"support%1:09:00::","call_ids":["semeval2013.d008.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"support%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":894,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005118999999999999,"input_uncached_usd":0.000044699999999999996,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.294650041963905}
{"item_id":"semeval2013.d008.s016.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003181,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.903944666963071}
{"item_id":"semeval2013.d008.s016.t002","gold_sense_keys":["recession%1:26:00::"],"candidates":[{"index":1,"sense_key":"recession%1:26:00::","synset_id":"recession.n.01"},{"index":2,"sense_key":"recession%1:25:00::","synset_id":"recess.n.02"},{"index":3,"sense_key":"recession%1:14:00::","synset_id":"recession.n.03"},{"index":4,"sense_key":"recession%1:04:01::","synset_id":"recession.n.04"},{"index":5,"sense_key":"recession%1:04:00::","synset_id":"receding.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recession%1:26:00::","call_ids":["semeval2013.d008.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recession%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007238,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.471617417002562}
{"item_id":"semeval2013.d008.s016.t003","gold_sense_keys":["recovery%1:11:00::"],"candidates":[{"index":1,"sense_key":"recovery%1:11:00::","synset_id":"recovery.n.01"},{"index":2,"sense_key":"recovery%1:22:00::","synset_id":"convalescence.n.01"},{"index":3,"sense_key":"recovery%1:04:00::","synset_id":"recovery.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recovery%1:11:00::","call_ids":["semeval2013.d008.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recovery%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013204999999999998,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.276584708946757}
{"item_id":"semeval2013.d008.s017.t001","gold_sense_keys":["safety_net%1:10:00::"],"candidates":[{"index":1,"sense_key":"safety_net%1:10:00::","synset_id":"safety_net.n.01"},{"index":2,"sense_key":"safety_net%1:06:00::","synset_id":"safety_net.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"safety_net%1:10:00::","call_ids":["semeval2013.d008.s017.t001__v1__a1","semeval2013.d008.s017.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"safety_net%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":923,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046855,"input_uncached_usd":0.00004615,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.337547749979421}
{"item_id":"semeval2013.d008.s018.t002","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"week%1:28:02::","call_ids":["semeval2013.d008.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"week%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023439999999999998,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8996950830332935}
{"item_id":"semeval2013.d008.s018.t003","gold_sense_keys":["proportion%1:24:00::"],"candidates":[{"index":1,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":2,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":3,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":4,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":5,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"proportion%1:24:01::","call_ids":["semeval2013.d008.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"proportion%1:24:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.03659695899114}
{"item_id":"semeval2013.d008.s018.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003155,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.950494375021663}
{"item_id":"semeval2013.d008.s019.t000","gold_sense_keys":["mood%1:12:00::","mood%1:26:00::"],"candidates":[{"index":1,"sense_key":"mood%1:12:00::","synset_id":"temper.n.02"},{"index":2,"sense_key":"mood%1:26:00::","synset_id":"climate.n.02"},{"index":3,"sense_key":"mood%1:24:00::","synset_id":"mood.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mood%1:26:00::","call_ids":["semeval2013.d008.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mood%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013289999999999998,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.222285458003171}
{"item_id":"semeval2013.d008.s019.t001","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"country%1:14:01::","call_ids":["semeval2013.d008.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"country%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00082945,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.696589500003029}
{"item_id":"semeval2013.d008.s019.t002","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041985,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.052524374972563}
{"item_id":"semeval2013.d008.s019.t003","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015375,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2864124579937197}
{"item_id":"semeval2013.d008.s020.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s020.t000__v1__a1","semeval2013.d008.s020.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004146499999999999,"input_uncached_usd":0.00004345,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.168275250005536}
{"item_id":"semeval2013.d008.s020.t003","gold_sense_keys":["president%1:18:00::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"president%1:18:00::","call_ids":["semeval2013.d008.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"president%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001623,"input_uncached_usd":0.000027899999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8458281659986824}
{"item_id":"semeval2013.d008.s020.t004","gold_sense_keys":["sense%1:09:05::"],"candidates":[{"index":1,"sense_key":"sense%1:09:05::","synset_id":"sense.n.01"},{"index":2,"sense_key":"sense%1:10:00::","synset_id":"sense.n.02"},{"index":3,"sense_key":"sense%1:09:02::","synset_id":"sense.n.03"},{"index":4,"sense_key":"sense%1:09:04::","synset_id":"common_sense.n.01"},{"index":5,"sense_key":"sense%1:09:06::","synset_id":"sense.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sense%1:09:05::","call_ids":["semeval2013.d008.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sense%1:09:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039295,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.962735957989935}
{"item_id":"semeval2013.d008.s020.t005","gold_sense_keys":["progress%1:04:01::"],"candidates":[{"index":1,"sense_key":"progress%1:04:01::","synset_id":"advancement.n.03"},{"index":2,"sense_key":"progress%1:04:00::","synset_id":"progress.n.02"},{"index":3,"sense_key":"progress%1:11:00::","synset_id":"progress.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"progress%1:04:01::","call_ids":["semeval2013.d008.s020.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"progress%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002337,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6497751659480855}
{"item_id":"semeval2013.d008.s021.t002","gold_sense_keys":["recession%1:26:00::"],"candidates":[{"index":1,"sense_key":"recession%1:26:00::","synset_id":"recession.n.01"},{"index":2,"sense_key":"recession%1:25:00::","synset_id":"recess.n.02"},{"index":3,"sense_key":"recession%1:14:00::","synset_id":"recession.n.03"},{"index":4,"sense_key":"recession%1:04:01::","synset_id":"recession.n.04"},{"index":5,"sense_key":"recession%1:04:00::","synset_id":"receding.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recession%1:26:00::","call_ids":["semeval2013.d008.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recession%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033935,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.358692791953217}
{"item_id":"semeval2013.d008.s021.t003","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028405,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.234939125017263}
{"item_id":"semeval2013.d008.s022.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2013.d008.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003005,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.190344000002369}
{"item_id":"semeval2013.d008.s022.t001","gold_sense_keys":["poll%1:09:00::"],"candidates":[{"index":1,"sense_key":"poll%1:09:00::","synset_id":"poll.n.01"},{"index":2,"sense_key":"poll%1:08:01::","synset_id":"pate.n.02"},{"index":3,"sense_key":"poll%1:08:02::","synset_id":"poll.n.03"},{"index":4,"sense_key":"poll%1:05:00::","synset_id":"poll.n.04"},{"index":5,"sense_key":"poll%1:04:00::","synset_id":"poll.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"poll%1:09:00::","call_ids":["semeval2013.d008.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"poll%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010849999999999998,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6002957920427434}
{"item_id":"semeval2013.d008.s022.t002","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respondent%1:18:01::","call_ids":["semeval2013.d008.s022.t002__v1__a1","semeval2013.d008.s022.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respondent%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005953499999999999,"input_uncached_usd":0.000044949999999999995,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.905119208968244}
{"item_id":"semeval2013.d008.s022.t004","gold_sense_keys":["whole%1:09:00::"],"candidates":[{"index":1,"sense_key":"whole%1:09:00::","synset_id":"whole.n.01"},{"index":2,"sense_key":"whole%1:03:00::","synset_id":"whole.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"whole%1:09:00::","call_ids":["semeval2013.d008.s022.t004__v1__a1","semeval2013.d008.s022.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"whole%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":947,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00054655,"input_uncached_usd":0.00004735,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.70748033397831}
{"item_id":"semeval2013.d008.s022.t005","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s022.t005__v1__a1","semeval2013.d008.s022.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":2658,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.00111005,"input_uncached_usd":0.00004685,"input_cached_usd":0.0,"output_usd":0.0010632,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.865724500035867}
{"item_id":"semeval2013.d008.s022.t007","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d008.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018389999999999997,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5869724589865655}
{"item_id":"semeval2013.d008.s022.t008","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s022.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007023,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.115955542016309}
{"item_id":"semeval2013.d008.s022.t009","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s022.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005386,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.70282262499677}
{"item_id":"semeval2013.d008.s023.t000","gold_sense_keys":["relative%1:18:00::"],"candidates":[{"index":1,"sense_key":"relative%1:18:00::","synset_id":"relative.n.01"},{"index":2,"sense_key":"relative%1:05:00::","synset_id":"relative.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"relative%1:18:00::","call_ids":["semeval2013.d008.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"relative%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004406,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.05600104102632}
{"item_id":"semeval2013.d008.s023.t001","gold_sense_keys":["victim%1:18:00::"],"candidates":[{"index":1,"sense_key":"victim%1:18:00::","synset_id":"victim.n.01"},{"index":2,"sense_key":"victim%1:18:01::","synset_id":"victim.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"victim%1:18:00::","call_ids":["semeval2013.d008.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"victim%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003873,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.717737416969612}
{"item_id":"semeval2013.d008.s023.t002","gold_sense_keys":["crime%1:04:00::"],"candidates":[{"index":1,"sense_key":"crime%1:04:00::","synset_id":"crime.n.01"},{"index":2,"sense_key":"crime%1:04:01::","synset_id":"crime.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crime%1:04:00::","call_ids":["semeval2013.d008.s023.t002__v1__a1","semeval2013.d008.s023.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crime%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1009,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00044724999999999995,"input_uncached_usd":0.00005044999999999999,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.840041874034796}
{"item_id":"semeval2013.d008.s023.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029174999999999996,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.772852167021483}
{"item_id":"semeval2013.d008.s023.t004","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000215,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9043116670218296}
{"item_id":"semeval2013.d008.s023.t005","gold_sense_keys":["figure%1:07:01::"],"candidates":[{"index":1,"sense_key":"figure%1:10:00::","synset_id":"figure.n.01"},{"index":2,"sense_key":"figure%1:08:00::","synset_id":"human_body.n.01"},{"index":3,"sense_key":"figure%1:23:00::","synset_id":"digit.n.01"},{"index":4,"sense_key":"figure%1:06:00::","synset_id":"figure.n.04"},{"index":5,"sense_key":"figure%1:18:00::","synset_id":"name.n.04"},{"index":6,"sense_key":"figure%1:25:00::","synset_id":"figure.n.06"},{"index":7,"sense_key":"figure%1:21:00::","synset_id":"figure.n.07"},{"index":8,"sense_key":"figure%1:07:00::","synset_id":"figure.n.08"},{"index":9,"sense_key":"figure%1:07:01::","synset_id":"number.n.01"},{"index":10,"sense_key":"figure%1:10:01::","synset_id":"trope.n.01"},{"index":11,"sense_key":"figure%1:09:00::","synset_id":"figure.n.11"},{"index":12,"sense_key":"figure%1:06:01::","synset_id":"design.n.04"},{"index":13,"sense_key":"figure%1:04:00::","synset_id":"figure.n.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"figure%1:07:01::","call_ids":["semeval2013.d008.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"figure%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":922,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025729999999999997,"input_uncached_usd":0.000046099999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.688520165975206}
{"item_id":"semeval2013.d008.s024.t001","gold_sense_keys":["support%1:04:00::","support%1:09:00::"],"candidates":[{"index":1,"sense_key":"support%1:04:04::","synset_id":"support.n.01"},{"index":2,"sense_key":"support%1:04:00::","synset_id":"support.n.02"},{"index":3,"sense_key":"support%1:09:00::","synset_id":"support.n.03"},{"index":4,"sense_key":"support%1:04:03::","synset_id":"support.n.04"},{"index":5,"sense_key":"support%1:04:02::","synset_id":"documentation.n.03"},{"index":6,"sense_key":"support%1:21:00::","synset_id":"support.n.06"},{"index":7,"sense_key":"support%1:06:01::","synset_id":"support.n.07"},{"index":8,"sense_key":"support%1:04:01::","synset_id":"support.n.08"},{"index":9,"sense_key":"support%1:10:01::","synset_id":"accompaniment.n.02"},{"index":10,"sense_key":"support%1:06:00::","synset_id":"support.n.10"},{"index":11,"sense_key":"support%1:21:01::","synset_id":"support.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"support%1:04:00::","call_ids":["semeval2013.d008.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"support%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.00099805,"input_uncached_usd":0.000044449999999999996,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.187264209031127}
{"item_id":"semeval2013.d008.s024.t004","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s024.t004__v1__a1","semeval2013.d008.s024.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":2273,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009560500000000001,"input_uncached_usd":0.00004685,"input_cached_usd":0.0,"output_usd":0.0009092,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.63792391604511}
{"item_id":"semeval2013.d008.s024.t007","gold_sense_keys":["development%1:04:01::","development%1:22:02::","development%1:26:00::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"development%1:22:02::","call_ids":["semeval2013.d008.s024.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"development%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":852,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005354,"input_uncached_usd":0.0000426,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.982257583003957}
{"item_id":"semeval2013.d008.s024.t008","gold_sense_keys":["chart%1:10:00::"],"candidates":[{"index":1,"sense_key":"chart%1:10:00::","synset_id":"chart.n.01"},{"index":2,"sense_key":"chart%1:06:00::","synset_id":"chart.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"chart%1:10:00::","call_ids":["semeval2013.d008.s024.t008__v1__a1","semeval2013.d008.s024.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"chart%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046705,"input_uncached_usd":0.00004465,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.823461708030663}
{"item_id":"semeval2013.d008.s025.t001","gold_sense_keys":["system%1:14:00::"],"candidates":[{"index":1,"sense_key":"system%1:06:00::","synset_id":"system.n.01"},{"index":2,"sense_key":"system%1:14:00::","synset_id":"system.n.02"},{"index":3,"sense_key":"system%1:27:00::","synset_id":"system.n.03"},{"index":4,"sense_key":"system%1:09:00::","synset_id":"system.n.04"},{"index":5,"sense_key":"system%1:09:02::","synset_id":"arrangement.n.03"},{"index":6,"sense_key":"system%1:08:00::","synset_id":"system.n.06"},{"index":7,"sense_key":"system%1:09:01::","synset_id":"system.n.07"},{"index":8,"sense_key":"system%1:08:01::","synset_id":"system.n.08"},{"index":9,"sense_key":"system%1:07:00::","synset_id":"organization.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"system%1:14:00::","call_ids":["semeval2013.d008.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"system%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":855,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045874999999999996,"input_uncached_usd":0.000042749999999999996,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.478908125020098}
{"item_id":"semeval2013.d008.s025.t002","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s025.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029495,"input_uncached_usd":0.00003255,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.434353958000429}
{"item_id":"semeval2013.d008.s025.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s025.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021544999999999998,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.729495958017651}
{"item_id":"semeval2013.d008.s026.t000","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"respondent%1:18:01::","call_ids":["semeval2013.d008.s026.t000__v1__a1","semeval2013.d008.s026.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"respondent%1:18:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1037,"cached_input_tokens":0,"output_tokens":1888,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00080705,"input_uncached_usd":0.00005185,"input_cached_usd":0.0,"output_usd":0.0007552,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.040802374947816}
{"item_id":"semeval2013.d008.s026.t002","gold_sense_keys":["view%1:09:02::","view%1:09:04::"],"candidates":[{"index":1,"sense_key":"view%1:09:02::","synset_id":"position.n.03"},{"index":2,"sense_key":"view%1:09:01::","synset_id":"view.n.02"},{"index":3,"sense_key":"view%1:04:00::","synset_id":"view.n.03"},{"index":4,"sense_key":"view%1:15:00::","synset_id":"view.n.04"},{"index":5,"sense_key":"view%1:09:04::","synset_id":"opinion.n.01"},{"index":6,"sense_key":"view%1:10:00::","synset_id":"opinion.n.02"},{"index":7,"sense_key":"view%1:09:00::","synset_id":"view.n.07"},{"index":8,"sense_key":"view%1:06:00::","synset_id":"scene.n.08"},{"index":9,"sense_key":"view%1:07:00::","synset_id":"horizon.n.02"},{"index":10,"sense_key":"view%1:07:01::","synset_id":"view.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"view%1:09:04::","call_ids":["semeval2013.d008.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"view%1:09:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056305,"input_uncached_usd":0.00004465,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.476691041025333}
{"item_id":"semeval2013.d008.s026.t004","gold_sense_keys":["president%1:18:00::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"president%1:18:00::","call_ids":["semeval2013.d008.s026.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"president%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021884999999999998,"input_uncached_usd":0.00003325,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7650789589970373}
{"item_id":"semeval2013.d008.s026.t005","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d008.s026.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":777,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027565,"input_uncached_usd":0.000038849999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.335960291035008}
{"item_id":"semeval2013.d008.s027.t000","gold_sense_keys":["brazil%1:15:00::"],"candidates":[{"index":1,"sense_key":"brazil%1:15:00::","synset_id":"brazil.n.01"},{"index":2,"sense_key":"brazil%1:13:00::","synset_id":"brazil_nut.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"brazil%1:15:00::","call_ids":["semeval2013.d008.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"brazil%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":530,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036569999999999994,"input_uncached_usd":0.0000265,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.814007374981884}
{"item_id":"semeval2013.d008.s027.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":687,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016874999999999998,"input_uncached_usd":0.00003435,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1850004160078242}
{"item_id":"semeval2013.d008.s027.t002","gold_sense_keys":["leadership%1:04:00::","leadership%1:09:00::","leadership%1:26:00::"],"candidates":[{"index":1,"sense_key":"leadership%1:04:00::","synset_id":"leadership.n.01"},{"index":2,"sense_key":"leadership%1:14:00::","synset_id":"leadership.n.02"},{"index":3,"sense_key":"leadership%1:26:00::","synset_id":"leadership.n.03"},{"index":4,"sense_key":"leadership%1:09:00::","synset_id":"leadership.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"leadership%1:09:00::","call_ids":["semeval2013.d008.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"leadership%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039365,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.525865582982078}
{"item_id":"semeval2013.d008.s027.t004","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s027.t004__v1__a1","semeval2013.d008.s027.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1061,"cached_input_tokens":0,"output_tokens":1824,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00078265,"input_uncached_usd":0.00005305,"input_cached_usd":0.0,"output_usd":0.0007296,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.90256887493888}
{"item_id":"semeval2013.d008.s027.t005","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005245,"input_uncached_usd":0.0000317,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.107142583001405}
{"item_id":"semeval2013.d008.s027.t006","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s027.t006__v1__a1","semeval2013.d008.s027.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1169,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00042964999999999995,"input_uncached_usd":0.00005845,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.148685208056122}
{"item_id":"semeval2013.d008.s027.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021725,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.788097500044387}
{"item_id":"semeval2013.d008.s027.t009","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d008.s027.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037084999999999993,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.483698875003029}
{"item_id":"semeval2013.d008.s028.t000","gold_sense_keys":["united_states%1:15:00::"],"candidates":[{"index":1,"sense_key":"united_states%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"united_states%1:14:00::","synset_id":"united_states_government.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"united_states%1:15:00::","call_ids":["semeval2013.d008.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"united_states%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033969999999999996,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.00161008402938}
{"item_id":"semeval2013.d008.s028.t001","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d008.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":648,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044839999999999995,"input_uncached_usd":0.0000324,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.292388208967168}
{"item_id":"semeval2013.d008.s028.t002","gold_sense_keys":["respondent%1:18:00::"],"candidates":[{"index":1,"sense_key":"respondent%1:18:01::","synset_id":"respondent.n.01"},{"index":2,"sense_key":"respondent%1:18:00::","synset_id":"respondent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"respondent%1:18:00::","call_ids":["semeval2013.d008.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"respondent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043939999999999995,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.813735458999872}
{"item_id":"semeval2013.d008.s028.t006","gold_sense_keys":["distinction%1:07:01::"],"candidates":[{"index":1,"sense_key":"distinction%1:09:00::","synset_id":"differentiation.n.01"},{"index":2,"sense_key":"distinction%1:26:00::","synset_id":"eminence.n.01"},{"index":3,"sense_key":"distinction%1:07:01::","synset_id":"distinction.n.03"},{"index":4,"sense_key":"distinction%1:07:00::","synset_id":"distinction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"distinction%1:26:00::","call_ids":["semeval2013.d008.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"distinction%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003926,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.681343167030718}
{"item_id":"semeval2013.d009.s000.t001","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.324812917038798}
{"item_id":"semeval2013.d009.s000.t003","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d009.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":2321,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009613499999999999,"input_uncached_usd":0.00003295,"input_cached_usd":0.0,"output_usd":0.0009283999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.677495083014946}
{"item_id":"semeval2013.d009.s000.t004","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1088,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00013759999999999998,"input_uncached_usd":0.000054399999999999994,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2706342919846065}
{"item_id":"semeval2013.d009.s001.t000","gold_sense_keys":["day%1:28:03::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"day%1:28:03::","call_ids":["semeval2013.d009.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"day%1:28:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022124999999999998,"input_uncached_usd":0.00003565,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.020732625038363}
{"item_id":"semeval2013.d009.s001.t001","gold_sense_keys":["supreme_court%1:14:00::"],"candidates":[{"index":1,"sense_key":"supreme_court%1:14:00::","synset_id":"supreme_court.n.01"},{"index":2,"sense_key":"supreme_court%1:14:01::","synset_id":"supreme_court.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"supreme_court%1:14:00::","call_ids":["semeval2013.d009.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"supreme_court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002543,"input_uncached_usd":0.0000175,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.140505084011238}
{"item_id":"semeval2013.d009.s001.t002","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d009.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057115,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.15383045800263}
{"item_id":"semeval2013.d009.s001.t005","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001371,"input_uncached_usd":0.0000283,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.384814291028306}
{"item_id":"semeval2013.d009.s001.t006","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010324999999999999,"input_uncached_usd":0.00002005,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4854250000207685}
{"item_id":"semeval2013.d009.s001.t007","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005356,"input_uncached_usd":0.000017199999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.986432708043139}
{"item_id":"semeval2013.d009.s001.t009","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1111,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00018994999999999998,"input_uncached_usd":0.000055549999999999995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.744640624965541}
{"item_id":"semeval2013.d009.s002.t002","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":441,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020764999999999998,"input_uncached_usd":0.00002205,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.765601792023517}
{"item_id":"semeval2013.d009.s002.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2013.d009.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004099,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.629043875029311}
{"item_id":"semeval2013.d009.s002.t004","gold_sense_keys":["petition%1:10:00::"],"candidates":[{"index":1,"sense_key":"petition%1:10:00::","synset_id":"request.n.01"},{"index":2,"sense_key":"petition%1:10:01::","synset_id":"prayer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"petition%1:10:00::","call_ids":["semeval2013.d009.s002.t004__v1__a1","semeval2013.d009.s002.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"petition%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00053685,"input_uncached_usd":0.000037649999999999994,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.936741292010993}
{"item_id":"semeval2013.d009.s002.t005","gold_sense_keys":["review%1:04:02::","review%1:04:03::","review%1:09:00::"],"candidates":[{"index":1,"sense_key":"review%1:09:00::","synset_id":"reappraisal.n.01"},{"index":2,"sense_key":"review%1:10:01::","synset_id":"review.n.02"},{"index":3,"sense_key":"review%1:04:00::","synset_id":"follow-up.n.03"},{"index":4,"sense_key":"review%1:21:00::","synset_id":"review.n.04"},{"index":5,"sense_key":"review%1:10:02::","synset_id":"revue.n.01"},{"index":6,"sense_key":"review%1:10:03::","synset_id":"review.n.06"},{"index":7,"sense_key":"review%1:10:00::","synset_id":"recapitulation.n.03"},{"index":8,"sense_key":"review%1:04:03::","synset_id":"review.n.08"},{"index":9,"sense_key":"review%1:04:01::","synset_id":"review.n.09"},{"index":10,"sense_key":"review%1:04:02::","synset_id":"inspection.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"review%1:04:03::","call_ids":["semeval2013.d009.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"review%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004237,"input_uncached_usd":0.000033299999999999996,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.884140042006038}
{"item_id":"semeval2013.d009.s003.t000","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1207,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00014355,"input_uncached_usd":0.000060349999999999996,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5738255829783157}
{"item_id":"semeval2013.d009.s003.t002","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":2,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":4,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suit%1:04:00::","call_ids":["semeval2013.d009.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014084999999999998,"input_uncached_usd":0.00003205,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0595898749888875}
{"item_id":"semeval2013.d009.s003.t003","gold_sense_keys":["state%1:15:01::"],"candidates":[{"index":1,"sense_key":"state%1:15:01::","synset_id":"state.n.01"},{"index":2,"sense_key":"state%1:03:00::","synset_id":"state.n.02"},{"index":3,"sense_key":"state%1:14:01::","synset_id":"state.n.03"},{"index":4,"sense_key":"state%1:14:00::","synset_id":"state.n.04"},{"index":5,"sense_key":"state%1:26:02::","synset_id":"state_of_matter.n.01"},{"index":6,"sense_key":"state%1:26:01::","synset_id":"state.n.06"},{"index":7,"sense_key":"state%1:15:00::","synset_id":"country.n.02"},{"index":8,"sense_key":"state%1:14:02::","synset_id":"department_of_state.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"state%1:15:01::","call_ids":["semeval2013.d009.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"state%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006576500000000001,"input_uncached_usd":0.00003685,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.876170750008896}
{"item_id":"semeval2013.d009.s003.t005","gold_sense_keys":["overhaul%1:04:01::"],"candidates":[{"index":1,"sense_key":"overhaul%1:04:00::","synset_id":"overhaul.n.01"},{"index":2,"sense_key":"overhaul%1:04:01::","synset_id":"renovation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"overhaul%1:04:01::","call_ids":["semeval2013.d009.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"overhaul%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000285,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.346893707988784}
{"item_id":"semeval2013.d009.s003.t006","gold_sense_keys":["congress%1:14:01::"],"candidates":[{"index":1,"sense_key":"congress%1:14:01::","synset_id":"congress.n.01"},{"index":2,"sense_key":"congress%1:14:00::","synset_id":"congress.n.02"},{"index":3,"sense_key":"congress%1:14:02::","synset_id":"congress.n.03"},{"index":4,"sense_key":"congress%1:04:01::","synset_id":"sexual_intercourse.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"congress%1:14:01::","call_ids":["semeval2013.d009.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"congress%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.000159,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.949438249983359}
{"item_id":"semeval2013.d009.s003.t007","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d009.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.000187,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9516473329858854}
{"item_id":"semeval2013.d009.s003.t008","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s003.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032115,"input_uncached_usd":0.00003315,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.611128208984155}
{"item_id":"semeval2013.d009.s003.t009","gold_sense_keys":["rallying_cry%1:10:00::"],"candidates":[{"index":1,"sense_key":"rallying_cry%1:10:00::","synset_id":"war_cry.n.01"},{"index":2,"sense_key":"rallying_cry%1:10:01::","synset_id":"war_cry.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rallying_cry%1:10:00::","call_ids":["semeval2013.d009.s003.t009__v1__a1","semeval2013.d009.s003.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rallying_cry%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":945,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00052085,"input_uncached_usd":0.000047249999999999997,"input_cached_usd":0.0,"output_usd":0.00047359999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.180093666014727}
{"item_id":"semeval2013.d009.s004.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":538,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001613,"input_uncached_usd":0.0000269,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.941182292008307}
{"item_id":"semeval2013.d009.s004.t001","gold_sense_keys":["suit%1:04:00::"],"candidates":[{"index":1,"sense_key":"suit%1:06:00::","synset_id":"suit.n.01"},{"index":2,"sense_key":"suit%1:04:00::","synset_id":"lawsuit.n.01"},{"index":3,"sense_key":"suit%1:18:00::","synset_id":"suit.n.03"},{"index":4,"sense_key":"suit%1:10:01::","synset_id":"courtship.n.01"},{"index":5,"sense_key":"suit%1:10:00::","synset_id":"suit.n.05"},{"index":6,"sense_key":"suit%1:06:01::","synset_id":"suit.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"suit%1:04:00::","call_ids":["semeval2013.d009.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"suit%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":680,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021959999999999997,"input_uncached_usd":0.000034,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9427152079879306}
{"item_id":"semeval2013.d009.s004.t002","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d009.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002127,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.130800666986033}
{"item_id":"semeval2013.d009.s004.t003","gold_sense_keys":["argument%1:10:00::"],"candidates":[{"index":1,"sense_key":"argument%1:10:02::","synset_id":"argument.n.01"},{"index":2,"sense_key":"argument%1:10:03::","synset_id":"controversy.n.01"},{"index":3,"sense_key":"argument%1:10:00::","synset_id":"argument.n.03"},{"index":4,"sense_key":"argument%1:10:01::","synset_id":"argument.n.04"},{"index":5,"sense_key":"argument%1:10:04::","synset_id":"argument.n.05"},{"index":6,"sense_key":"argument%1:09:00::","synset_id":"argument.n.06"},{"index":7,"sense_key":"argument%1:09:01::","synset_id":"argumentation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"argument%1:10:00::","call_ids":["semeval2013.d009.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"argument%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037569999999999997,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.395029208040796}
{"item_id":"semeval2013.d009.s004.t004","gold_sense_keys":["march%1:28:00::"],"candidates":[{"index":1,"sense_key":"march%1:28:00::","synset_id":"march.n.01"},{"index":2,"sense_key":"march%1:04:00::","synset_id":"march.n.02"},{"index":3,"sense_key":"march%1:04:02::","synset_id":"march.n.03"},{"index":4,"sense_key":"march%1:14:00::","synset_id":"march.n.04"},{"index":5,"sense_key":"march%1:15:00::","synset_id":"borderland.n.01"},{"index":6,"sense_key":"march%1:10:01::","synset_id":"marching_music.n.01"},{"index":7,"sense_key":"march%1:10:00::","synset_id":"master_of_architecture.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"march%1:28:00::","call_ids":["semeval2013.d009.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"march%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004232,"input_uncached_usd":0.0000328,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.137839041999541}
{"item_id":"semeval2013.d009.s004.t006","gold_sense_keys":["race%1:11:01::"],"candidates":[{"index":1,"sense_key":"race%1:11:01::","synset_id":"race.n.01"},{"index":2,"sense_key":"race%1:11:00::","synset_id":"race.n.02"},{"index":3,"sense_key":"race%1:14:00::","synset_id":"race.n.03"},{"index":4,"sense_key":"race%1:14:01::","synset_id":"subspecies.n.01"},{"index":5,"sense_key":"race%1:19:00::","synset_id":"slipstream.n.01"},{"index":6,"sense_key":"race%1:06:00::","synset_id":"raceway.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"race%1:11:01::","call_ids":["semeval2013.d009.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"race%1:11:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021585,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7500690419692546}
{"item_id":"semeval2013.d009.s004.t007","gold_sense_keys":["swing%1:26:01::"],"candidates":[{"index":1,"sense_key":"swing%1:26:01::","synset_id":"swing.n.01"},{"index":2,"sense_key":"swing%1:06:00::","synset_id":"swing.n.02"},{"index":3,"sense_key":"swing%1:04:06::","synset_id":"swing.n.03"},{"index":4,"sense_key":"swing%1:04:00::","synset_id":"swing.n.04"},{"index":5,"sense_key":"swing%1:10:01::","synset_id":"swing.n.05"},{"index":6,"sense_key":"swing%1:07:00::","synset_id":"lilt.n.01"},{"index":7,"sense_key":"swing%1:04:03::","synset_id":"golf_stroke.n.01"},{"index":8,"sense_key":"swing%1:04:02::","synset_id":"baseball_swing.n.01"},{"index":9,"sense_key":"swing%1:04:04::","synset_id":"swing.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"swing%1:26:01::","call_ids":["semeval2013.d009.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"swing%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022199999999999998,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7136268750182353}
{"item_id":"semeval2013.d009.s004.t009","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s004.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":1681,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007132999999999998,"input_uncached_usd":0.0000409,"input_cached_usd":0.0,"output_usd":0.0006724,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.638814958976582}
{"item_id":"semeval2013.d009.s004.t010","gold_sense_keys":["decision%1:09:00::"],"candidates":[{"index":1,"sense_key":"decision%1:04:00::","synset_id":"decision.n.01"},{"index":2,"sense_key":"decision%1:09:00::","synset_id":"decision.n.02"},{"index":3,"sense_key":"decision%1:11:02::","synset_id":"decision.n.03"},{"index":4,"sense_key":"decision%1:11:01::","synset_id":"decision.n.04"},{"index":5,"sense_key":"decision%1:07:00::","synset_id":"decisiveness.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"decision%1:09:00::","call_ids":["semeval2013.d009.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"decision%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003182,"input_uncached_usd":0.0000302,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.943067999964114}
{"item_id":"semeval2013.d009.s005.t001","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d009.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":858,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004333,"input_uncached_usd":0.0000429,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.889406458009034}
{"item_id":"semeval2013.d009.s005.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005299500000000001,"input_uncached_usd":0.000037149999999999995,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.360536249994766}
{"item_id":"semeval2013.d009.s006.t003","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s006.t003__v1__a1","semeval2013.d009.s006.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1115,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00055495,"input_uncached_usd":0.00005575,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.5614658750128}
{"item_id":"semeval2013.d009.s007.t001","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005173,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.956784457957838}
{"item_id":"semeval2013.d009.s007.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":713,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027245,"input_uncached_usd":0.00003565,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.133055959013291}
{"item_id":"semeval2013.d009.s008.t000","gold_sense_keys":["sponsor%1:18:00::"],"candidates":[{"index":1,"sense_key":"sponsor%1:18:00::","synset_id":"patron.n.03"},{"index":2,"sense_key":"sponsor%1:18:01::","synset_id":"presenter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sponsor%1:18:00::","call_ids":["semeval2013.d009.s008.t000__v1__a1","semeval2013.d009.s008.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sponsor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":971,"cached_input_tokens":0,"output_tokens":736,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00034294999999999996,"input_uncached_usd":0.00004855,"input_cached_usd":0.0,"output_usd":0.0002944,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.514861874980852}
{"item_id":"semeval2013.d009.s008.t001","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":2,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":3,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":4,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":6,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":7,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d009.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":635,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016615,"input_uncached_usd":0.00003175,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5158834169851616}
{"item_id":"semeval2013.d009.s008.t002","gold_sense_keys":["stake%1:21:02::"],"candidates":[{"index":1,"sense_key":"stake%1:21:02::","synset_id":"interest.n.05"},{"index":2,"sense_key":"stake%1:10:00::","synset_id":"post.n.09"},{"index":3,"sense_key":"stake%1:06:01::","synset_id":"stake.n.03"},{"index":4,"sense_key":"stake%1:21:00::","synset_id":"stake.n.04"},{"index":5,"sense_key":"stake%1:06:00::","synset_id":"stake.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"stake%1:21:02::","call_ids":["semeval2013.d009.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"stake%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029265,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.533253249945119}
{"item_id":"semeval2013.d009.s009.t000","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":1298,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005428499999999999,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.0005191999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.455048667034134}
{"item_id":"semeval2013.d009.s009.t001","gold_sense_keys":["washington%1:15:01::"],"candidates":[{"index":1,"sense_key":"washington%1:15:01::","synset_id":"washington.n.01"},{"index":2,"sense_key":"washington%1:15:00::","synset_id":"washington.n.02"},{"index":3,"sense_key":"washington%1:14:00::","synset_id":"capital.n.06"},{"index":4,"sense_key":"washington%1:18:00::","synset_id":"washington.n.04"},{"index":5,"sense_key":"washington%1:18:01::","synset_id":"washington.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"washington%1:15:01::","call_ids":["semeval2013.d009.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"washington%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":622,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005495000000000001,"input_uncached_usd":0.0000311,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.005962916999124}
{"item_id":"semeval2013.d009.s009.t003","gold_sense_keys":["hour%1:28:00::"],"candidates":[{"index":1,"sense_key":"hour%1:28:00::","synset_id":"hour.n.01"},{"index":2,"sense_key":"hour%1:28:01::","synset_id":"hour.n.02"},{"index":3,"sense_key":"hour%1:28:02::","synset_id":"hour.n.03"},{"index":4,"sense_key":"hour%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hour%1:28:00::","call_ids":["semeval2013.d009.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hour%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031475000000000003,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.260243375028949}
{"item_id":"semeval2013.d009.s009.t004","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"court%1:14:00::","call_ids":["semeval2013.d009.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"court%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":811,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00084055,"input_uncached_usd":0.000040549999999999996,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.275946207984816}
{"item_id":"semeval2013.d009.s009.t005","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"conference%1:10:00::","call_ids":["semeval2013.d009.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"conference%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018399999999999997,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.073498166981153}
{"item_id":"semeval2013.d009.s009.t006","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1239,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00017075,"input_uncached_usd":0.00006195,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.171157459029928}
{"item_id":"semeval2013.d009.s010.t000","gold_sense_keys":["attendance%1:04:00::"],"candidates":[{"index":1,"sense_key":"attendance%1:04:00::","synset_id":"attendance.n.01"},{"index":2,"sense_key":"attendance%1:28:00::","synset_id":"attendance.n.02"},{"index":3,"sense_key":"attendance%1:14:00::","synset_id":"attendance.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attendance%1:04:00::","call_ids":["semeval2013.d009.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attendance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013079999999999998,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.472833041974809}
{"item_id":"semeval2013.d009.s010.t001","gold_sense_keys":["senate%1:14:01::"],"candidates":[{"index":1,"sense_key":"senate%1:14:00::","synset_id":"senate.n.01"},{"index":2,"sense_key":"senate%1:14:01::","synset_id":"united_states_senate.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"senate%1:14:01::","call_ids":["semeval2013.d009.s010.t001__v1__a1","semeval2013.d009.s010.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"senate%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":835,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00066895,"input_uncached_usd":0.00004175,"input_cached_usd":0.0,"output_usd":0.0006272,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.898671167029534}
{"item_id":"semeval2013.d009.s010.t002","gold_sense_keys":["opponent%1:18:00::"],"candidates":[{"index":1,"sense_key":"opponent%1:18:02::","synset_id":"opposition.n.04"},{"index":2,"sense_key":"opponent%1:18:00::","synset_id":"adversary.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"opponent%1:18:00::","call_ids":["semeval2013.d009.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"opponent%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043605,"input_uncached_usd":0.00002005,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0153997499728575}
{"item_id":"semeval2013.d009.s011.t000","gold_sense_keys":["guest%1:18:00::"],"candidates":[{"index":1,"sense_key":"guest%1:18:00::","synset_id":"guest.n.01"},{"index":2,"sense_key":"guest%1:18:02::","synset_id":"guest.n.02"},{"index":3,"sense_key":"guest%1:18:01::","synset_id":"guest.n.03"},{"index":4,"sense_key":"guest%1:06:00::","synset_id":"node.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"guest%1:18:00::","call_ids":["semeval2013.d009.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"guest%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020805,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.719426666968502}
{"item_id":"semeval2013.d009.s011.t001","gold_sense_keys":["dinner%1:14:00::"],"candidates":[{"index":1,"sense_key":"dinner%1:13:00::","synset_id":"dinner.n.01"},{"index":2,"sense_key":"dinner%1:14:00::","synset_id":"dinner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dinner%1:14:00::","call_ids":["semeval2013.d009.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dinner%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046189999999999995,"input_uncached_usd":0.0000203,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.785143584012985}
{"item_id":"semeval2013.d009.s012.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010494999999999999,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2742930829990655}
{"item_id":"semeval2013.d009.s012.t001","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005156000000000001,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.524635459005367}
{"item_id":"semeval2013.d009.s012.t002","gold_sense_keys":["years%1:28:01::"],"candidates":[{"index":1,"sense_key":"years%1:28:00::","synset_id":"old_age.n.01"},{"index":2,"sense_key":"years%1:28:01::","synset_id":"long_time.n.01"},{"index":3,"sense_key":"years%1:28:02::","synset_id":"days.n.01"},{"index":4,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":5,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":6,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":7,"sense_key":"class%1:14:04::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"years%1:28:01::","call_ids":["semeval2013.d009.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"years%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001914,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.399336167029105}
{"item_id":"semeval2013.d009.s013.t002","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d009.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":826,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0002013,"input_uncached_usd":0.0000413,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.685988000012003}
{"item_id":"semeval2013.d009.s014.t001","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010644999999999999,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.763069375010673}
{"item_id":"semeval2013.d009.s014.t003","gold_sense_keys":["action%1:04:02::"],"candidates":[{"index":1,"sense_key":"action%1:04:02::","synset_id":"action.n.01"},{"index":2,"sense_key":"action%1:26:00::","synset_id":"action.n.02"},{"index":3,"sense_key":"action%1:04:03::","synset_id":"military_action.n.01"},{"index":4,"sense_key":"action%1:22:00::","synset_id":"natural_process.n.01"},{"index":5,"sense_key":"action%1:10:00::","synset_id":"action.n.05"},{"index":6,"sense_key":"action%1:07:00::","synset_id":"action.n.06"},{"index":7,"sense_key":"action%1:06:00::","synset_id":"action.n.07"},{"index":8,"sense_key":"action%1:04:01::","synset_id":"legal_action.n.01"},{"index":9,"sense_key":"action%1:04:04::","synset_id":"action.n.09"},{"index":10,"sense_key":"action%1:04:00::","synset_id":"action.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"action%1:04:02::","call_ids":["semeval2013.d009.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"action%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007364499999999999,"input_uncached_usd":0.00003845,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.517932374961674}
{"item_id":"semeval2013.d009.s014.t004","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010639999999999998,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2195916660130024}
{"item_id":"semeval2013.d009.s015.t000","gold_sense_keys":["code%1:10:01::"],"candidates":[{"index":1,"sense_key":"code%1:10:01::","synset_id":"code.n.01"},{"index":2,"sense_key":"code%1:10:00::","synset_id":"code.n.02"},{"index":3,"sense_key":"code%1:10:02::","synset_id":"code.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"code%1:10:01::","call_ids":["semeval2013.d009.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"code%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036114999999999997,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.262478500022553}
{"item_id":"semeval2013.d009.s015.t002","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d009.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006157,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.984452458040323}
{"item_id":"semeval2013.d009.s015.t003","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026225,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.012574749998748}
{"item_id":"semeval2013.d009.s015.t004","gold_sense_keys":["organization%1:14:00::","organization%1:14:01::"],"candidates":[{"index":1,"sense_key":"organization%1:14:00::","synset_id":"organization.n.01"},{"index":2,"sense_key":"organization%1:09:00::","synset_id":"arrangement.n.03"},{"index":3,"sense_key":"organization%1:14:01::","synset_id":"administration.n.02"},{"index":4,"sense_key":"organization%1:04:00::","synset_id":"organization.n.04"},{"index":5,"sense_key":"organization%1:07:00::","synset_id":"organization.n.05"},{"index":6,"sense_key":"organization%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organization%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"organization%1:14:01::","call_ids":["semeval2013.d009.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"organization%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":711,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032355000000000003,"input_uncached_usd":0.00003555,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.055088374996558}
{"item_id":"semeval2013.d009.s015.t005","gold_sense_keys":["judge%1:18:00::"],"candidates":[{"index":1,"sense_key":"judge%1:18:00::","synset_id":"judge.n.01"},{"index":2,"sense_key":"judge%1:18:01::","synset_id":"evaluator.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"judge%1:18:00::","call_ids":["semeval2013.d009.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"judge%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005641,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.457282832998317}
{"item_id":"semeval2013.d009.s015.t006","gold_sense_keys":["speaker%1:18:00::"],"candidates":[{"index":1,"sense_key":"speaker%1:18:00::","synset_id":"speaker.n.01"},{"index":2,"sense_key":"speaker%1:06:00::","synset_id":"loudspeaker.n.01"},{"index":3,"sense_key":"speaker%1:18:01::","synset_id":"speaker.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"speaker%1:18:00::","call_ids":["semeval2013.d009.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"speaker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002365,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.0716272499994375}
{"item_id":"semeval2013.d009.s015.t008","gold_sense_keys":["program%1:10:00::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"program%1:10:00::","call_ids":["semeval2013.d009.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"program%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040114999999999997,"input_uncached_usd":0.000036349999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.050774125033058}
{"item_id":"semeval2013.d009.s015.t009","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041585,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.576983499980997}
{"item_id":"semeval2013.d009.s016.t000","gold_sense_keys":["proximity%1:07:00::"],"candidates":[{"index":1,"sense_key":"proximity%1:07:00::","synset_id":"proximity.n.01"},{"index":2,"sense_key":"proximity%1:15:00::","synset_id":"proximity.n.02"},{"index":3,"sense_key":"proximity%1:09:00::","synset_id":"proximity.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proximity%1:07:00::","call_ids":["semeval2013.d009.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proximity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003116,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.320469375001267}
{"item_id":"semeval2013.d009.s016.t002","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1216,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00024639999999999997,"input_uncached_usd":0.0000608,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6752428750041872}
{"item_id":"semeval2013.d009.s016.t003","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d009.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045135,"input_uncached_usd":0.00003535,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.4649887080304325}
{"item_id":"semeval2013.d009.s016.t004","gold_sense_keys":["interest%1:21:03::"],"candidates":[{"index":1,"sense_key":"interest%1:09:00::","synset_id":"interest.n.01"},{"index":2,"sense_key":"interest%1:07:01::","synset_id":"sake.n.01"},{"index":3,"sense_key":"interest%1:07:02::","synset_id":"interest.n.03"},{"index":4,"sense_key":"interest%1:21:00::","synset_id":"interest.n.04"},{"index":5,"sense_key":"interest%1:21:03::","synset_id":"interest.n.05"},{"index":6,"sense_key":"interest%1:14:00::","synset_id":"interest.n.06"},{"index":7,"sense_key":"interest%1:04:01::","synset_id":"pastime.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"interest%1:21:03::","call_ids":["semeval2013.d009.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"interest%1:21:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":683,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019414999999999998,"input_uncached_usd":0.000034149999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2192353749996983}
{"item_id":"semeval2013.d009.s017.t000","gold_sense_keys":["breach%1:04:01::"],"candidates":[{"index":1,"sense_key":"breach%1:04:01::","synset_id":"breach.n.01"},{"index":2,"sense_key":"breach%1:17:00::","synset_id":"breach.n.02"},{"index":3,"sense_key":"breach%1:11:00::","synset_id":"rupture.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"breach%1:04:01::","call_ids":["semeval2013.d009.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"breach%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023524999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5712150420295075}
{"item_id":"semeval2013.d009.s017.t002","gold_sense_keys":["indifference%1:07:01::"],"candidates":[{"index":1,"sense_key":"indifference%1:12:00::","synset_id":"indifference.n.01"},{"index":2,"sense_key":"indifference%1:12:01::","synset_id":"emotionlessness.n.01"},{"index":3,"sense_key":"indifference%1:07:00::","synset_id":"apathy.n.02"},{"index":4,"sense_key":"indifference%1:07:01::","synset_id":"nonchalance.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"indifference%1:07:01::","call_ids":["semeval2013.d009.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"indifference%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":3856,"reasoning_output_tokens":3840},"cost":{"currency":"USD","total_usd":0.00156875,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0015424,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":22.08396412502043}
{"item_id":"semeval2013.d009.s017.t003","gold_sense_keys":["code%1:10:01::"],"candidates":[{"index":1,"sense_key":"code%1:10:01::","synset_id":"code.n.01"},{"index":2,"sense_key":"code%1:10:00::","synset_id":"code.n.02"},{"index":3,"sense_key":"code%1:10:02::","synset_id":"code.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"code%1:10:01::","call_ids":["semeval2013.d009.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"code%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.000158,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1484091660240665}
{"item_id":"semeval2013.d009.s017.t004","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d009.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024185,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3339759999653324}
{"item_id":"semeval2013.d009.s017.t005","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021165,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.468695791962091}
{"item_id":"semeval2013.d009.s017.t006","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"court%1:14:01::","call_ids":["semeval2013.d009.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"court%1:14:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":801,"cached_input_tokens":0,"output_tokens":2448,"reasoning_output_tokens":2432},"cost":{"currency":"USD","total_usd":0.00101925,"input_uncached_usd":0.00004005,"input_cached_usd":0.0,"output_usd":0.0009792,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.004456666996703}
{"item_id":"semeval2013.d009.s017.t007","gold_sense_keys":["rule%1:10:00::"],"candidates":[{"index":1,"sense_key":"rule%1:09:00::","synset_id":"rule.n.01"},{"index":2,"sense_key":"rule%1:09:01::","synset_id":"convention.n.02"},{"index":3,"sense_key":"rule%1:10:00::","synset_id":"rule.n.03"},{"index":4,"sense_key":"rule%1:10:01::","synset_id":"rule.n.04"},{"index":5,"sense_key":"rule%1:09:03::","synset_id":"principle.n.01"},{"index":6,"sense_key":"rule%1:28:00::","synset_id":"rule.n.06"},{"index":7,"sense_key":"rule%1:26:00::","synset_id":"dominion.n.01"},{"index":8,"sense_key":"rule%1:10:02::","synset_id":"rule.n.08"},{"index":9,"sense_key":"rule%1:10:03::","synset_id":"rule.n.09"},{"index":10,"sense_key":"rule%1:09:02::","synset_id":"principle.n.04"},{"index":11,"sense_key":"rule%1:09:04::","synset_id":"rule.n.11"},{"index":12,"sense_key":"rule%1:06:00::","synset_id":"rule.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"rule%1:10:00::","call_ids":["semeval2013.d009.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"rule%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035955,"input_uncached_usd":0.00004595,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.968878000043333}
{"item_id":"semeval2013.d009.s017.t008","gold_sense_keys":["president%1:18:01::"],"candidates":[{"index":1,"sense_key":"president%1:18:01::","synset_id":"president.n.01"},{"index":2,"sense_key":"president%1:18:04::","synset_id":"president_of_the_united_states.n.01"},{"index":3,"sense_key":"president%1:18:00::","synset_id":"president.n.03"},{"index":4,"sense_key":"president%1:18:03::","synset_id":"president.n.04"},{"index":5,"sense_key":"president%1:18:02::","synset_id":"president.n.05"},{"index":6,"sense_key":"president%1:04:04::","synset_id":"president_of_the_united_states.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"president%1:18:03::","call_ids":["semeval2013.d009.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"president%1:18:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004208,"input_uncached_usd":0.0000304,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.671328582975548}
{"item_id":"semeval2013.d009.s018.t000","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010939999999999999,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2710375000024214}
{"item_id":"semeval2013.d009.s018.t001","gold_sense_keys":["fundraiser%1:11:00::"],"candidates":[{"index":1,"sense_key":"fundraiser%1:18:00::","synset_id":"fundraiser.n.01"},{"index":2,"sense_key":"fundraiser%1:11:00::","synset_id":"fundraiser.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fundraiser%1:11:00::","call_ids":["semeval2013.d009.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fundraiser%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005908500000000001,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.943769000004977}
{"item_id":"semeval2013.d009.s018.t003","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1233,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00017045,"input_uncached_usd":0.00006165,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3678435829933733}
{"item_id":"semeval2013.d009.s018.t004","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d009.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":804,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005589999999999999,"input_uncached_usd":0.0000402,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.403156250016764}
{"item_id":"semeval2013.d009.s019.t000","gold_sense_keys":["appearance%1:04:00::"],"candidates":[{"index":1,"sense_key":"appearance%1:07:00::","synset_id":"appearance.n.01"},{"index":2,"sense_key":"appearance%1:11:00::","synset_id":"appearance.n.02"},{"index":3,"sense_key":"appearance%1:04:01::","synset_id":"appearance.n.03"},{"index":4,"sense_key":"appearance%1:09:00::","synset_id":"appearance.n.04"},{"index":5,"sense_key":"appearance%1:04:00::","synset_id":"appearance.n.05"},{"index":6,"sense_key":"appearance%1:04:02::","synset_id":"appearance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"appearance%1:04:00::","call_ids":["semeval2013.d009.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"appearance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002933,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.885771750006825}
{"item_id":"semeval2013.d009.s019.t001","gold_sense_keys":["assistance%1:04:00::"],"candidates":[{"index":1,"sense_key":"assistance%1:04:00::","synset_id":"aid.n.02"},{"index":2,"sense_key":"assistance%1:07:00::","synset_id":"aid.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assistance%1:04:00::","call_ids":["semeval2013.d009.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assistance%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028675,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.478296916000545}
{"item_id":"semeval2013.d009.s019.t002","gold_sense_keys":["event%1:03:00::"],"candidates":[{"index":1,"sense_key":"event%1:03:00::","synset_id":"event.n.01"},{"index":2,"sense_key":"event%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"event%1:19:00::","synset_id":"event.n.03"},{"index":4,"sense_key":"event%1:19:01::","synset_id":"consequence.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"event%1:03:00::","call_ids":["semeval2013.d009.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"event%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005467,"input_uncached_usd":0.0000283,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.162847790983506}
{"item_id":"semeval2013.d009.s019.t003","gold_sense_keys":["claim%1:10:02::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"claim%1:10:02::","call_ids":["semeval2013.d009.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"claim%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":629,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019145,"input_uncached_usd":0.00003145,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2305359999882057}
{"item_id":"semeval2013.d009.s020.t000","gold_sense_keys":["regard%1:09:01::"],"candidates":[{"index":1,"sense_key":"regard%1:09:00::","synset_id":"respect.n.01"},{"index":2,"sense_key":"regard%1:09:01::","synset_id":"attentiveness.n.01"},{"index":3,"sense_key":"regard%1:10:00::","synset_id":"regard.n.03"},{"index":4,"sense_key":"regard%1:04:00::","synset_id":"gaze.n.01"},{"index":5,"sense_key":"regard%1:26:00::","synset_id":"esteem.n.01"},{"index":6,"sense_key":"regard%1:12:02::","synset_id":"regard.n.06"},{"index":7,"sense_key":"regard%1:09:02::","synset_id":"respect.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"regard%1:09:02::","call_ids":["semeval2013.d009.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"regard%1:09:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003235,"input_uncached_usd":0.0000351,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.447192166990135}
{"item_id":"semeval2013.d009.s020.t002","gold_sense_keys":["business%1:09:01::"],"candidates":[{"index":1,"sense_key":"business%1:14:00::","synset_id":"business.n.01"},{"index":2,"sense_key":"business%1:04:01::","synset_id":"commercial_enterprise.n.02"},{"index":3,"sense_key":"business%1:04:00::","synset_id":"occupation.n.01"},{"index":4,"sense_key":"business%1:09:01::","synset_id":"business.n.04"},{"index":5,"sense_key":"business%1:09:00::","synset_id":"business.n.05"},{"index":6,"sense_key":"business%1:04:02::","synset_id":"business.n.06"},{"index":7,"sense_key":"business%1:14:02::","synset_id":"business.n.07"},{"index":8,"sense_key":"business%1:14:01::","synset_id":"clientele.n.01"},{"index":9,"sense_key":"business%1:04:03::","synset_id":"business.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"business%1:09:01::","call_ids":["semeval2013.d009.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"business%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003772499999999999,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.196165542001836}
{"item_id":"semeval2013.d009.s020.t003","gold_sense_keys":["court%1:14:00::"],"candidates":[{"index":1,"sense_key":"court%1:14:00::","synset_id":"court.n.01"},{"index":2,"sense_key":"court%1:06:02::","synset_id":"court.n.02"},{"index":3,"sense_key":"court%1:14:01::","synset_id":"court.n.03"},{"index":4,"sense_key":"court%1:06:01::","synset_id":"court.n.04"},{"index":5,"sense_key":"court%1:18:00::","synset_id":"court.n.05"},{"index":6,"sense_key":"court%1:14:02::","synset_id":"court.n.06"},{"index":7,"sense_key":"court%1:06:03::","synset_id":"motor_hotel.n.01"},{"index":8,"sense_key":"court%1:06:05::","synset_id":"court.n.08"},{"index":9,"sense_key":"court%1:06:04::","synset_id":"court.n.09"},{"index":10,"sense_key":"court%1:06:00::","synset_id":"court.n.10"},{"index":11,"sense_key":"court%1:04:00::","synset_id":"court.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"court%1:06:05::","call_ids":["semeval2013.d009.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"court%1:06:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":818,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005593,"input_uncached_usd":0.0000409,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.534463791991584}
{"item_id":"semeval2013.d009.s020.t004","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2013.d009.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":274,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013475,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.0001096,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.654377333004959}
{"item_id":"semeval2013.d009.s022.t001","gold_sense_keys":["justice%1:18:00::"],"candidates":[{"index":1,"sense_key":"justice%1:07:00::","synset_id":"justice.n.01"},{"index":2,"sense_key":"justice%1:04:00::","synset_id":"justice.n.02"},{"index":3,"sense_key":"justice%1:18:00::","synset_id":"judge.n.01"},{"index":4,"sense_key":"justice%1:14:00::","synset_id":"department_of_justice.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"justice%1:18:00::","call_ids":["semeval2013.d009.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"justice%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013309999999999998,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6656091249897145}
{"item_id":"semeval2013.d009.s022.t002","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d009.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":876,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007157999999999999,"input_uncached_usd":0.0000438,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.447439333016519}
{"item_id":"semeval2013.d009.s023.t001","gold_sense_keys":["administration%1:28:00::"],"candidates":[{"index":1,"sense_key":"administration%1:04:00::","synset_id":"administration.n.01"},{"index":2,"sense_key":"administration%1:14:00::","synset_id":"administration.n.02"},{"index":3,"sense_key":"administration%1:04:01::","synset_id":"administration.n.03"},{"index":4,"sense_key":"administration%1:28:00::","synset_id":"presidency.n.01"},{"index":5,"sense_key":"administration%1:04:04::","synset_id":"government.n.02"},{"index":6,"sense_key":"administration%1:04:02::","synset_id":"administration.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"administration%1:28:00::","call_ids":["semeval2013.d009.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"administration%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003949,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.10210883297259}
{"item_id":"semeval2013.d009.s023.t003","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019189999999999998,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6570392919820733}
{"item_id":"semeval2013.d009.s023.t005","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"level%1:26:00::","call_ids":["semeval2013.d009.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"level%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024524999999999997,"input_uncached_usd":0.00003405,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.213407167000696}
{"item_id":"semeval2013.d009.s024.t001","gold_sense_keys":["case%1:04:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2013.d009.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1168,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00014159999999999997,"input_uncached_usd":0.000058399999999999997,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1800363340298645}
{"item_id":"semeval2013.d009.s024.t002","gold_sense_keys":["law%1:10:00::"],"candidates":[{"index":1,"sense_key":"law%1:14:00::","synset_id":"law.n.01"},{"index":2,"sense_key":"law%1:10:00::","synset_id":"law.n.02"},{"index":3,"sense_key":"law%1:09:00::","synset_id":"law.n.03"},{"index":4,"sense_key":"law%1:09:01::","synset_id":"law.n.04"},{"index":5,"sense_key":"law%1:09:02::","synset_id":"jurisprudence.n.01"},{"index":6,"sense_key":"law%1:04:00::","synset_id":"law.n.06"},{"index":7,"sense_key":"law%1:14:01::","synset_id":"police.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"law%1:10:00::","call_ids":["semeval2013.d009.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"law%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004728499999999999,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.283679667045362}
{"item_id":"semeval2013.d009.s025.t000","gold_sense_keys":["indication%1:10:00::"],"candidates":[{"index":1,"sense_key":"indication%1:10:00::","synset_id":"indication.n.01"},{"index":2,"sense_key":"indication%1:10:01::","synset_id":"indication.n.02"},{"index":3,"sense_key":"indication%1:10:02::","synset_id":"indication.n.03"},{"index":4,"sense_key":"indication%1:10:03::","synset_id":"indication.n.04"},{"index":5,"sense_key":"indication%1:09:00::","synset_id":"reading.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"indication%1:10:02::","call_ids":["semeval2013.d009.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"indication%1:10:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.000237,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.435349249979481}
{"item_id":"semeval2013.d010.s000.t000","gold_sense_keys":["familiarity%1:09:00::"],"candidates":[{"index":1,"sense_key":"familiarity%1:09:00::","synset_id":"acquaintance.n.01"},{"index":2,"sense_key":"familiarity%1:07:01::","synset_id":"familiarity.n.02"},{"index":3,"sense_key":"familiarity%1:07:02::","synset_id":"familiarity.n.03"},{"index":4,"sense_key":"familiarity%1:07:03::","synset_id":"casualness.n.01"},{"index":5,"sense_key":"familiarity%1:04:00::","synset_id":"familiarity.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"familiarity%1:09:00::","call_ids":["semeval2013.d010.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"familiarity%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":355,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030575000000000003,"input_uncached_usd":0.000017749999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.121066208987031}
{"item_id":"semeval2013.d010.s001.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015269999999999997,"input_uncached_usd":0.000018299999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.809748707979452}
{"item_id":"semeval2013.d010.s001.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006517000000000001,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.134959582996089}
{"item_id":"semeval2013.d010.s002.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000223,"input_uncached_usd":0.000037,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9228684999980032}
{"item_id":"semeval2013.d010.s002.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023129999999999998,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7794321249821223}
{"item_id":"semeval2013.d010.s002.t003","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003207,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.795046624960378}
{"item_id":"semeval2013.d010.s003.t002","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002337,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5691632500384003}
{"item_id":"semeval2013.d010.s003.t004","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057915,"input_uncached_usd":0.00003515,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.473580499994569}
{"item_id":"semeval2013.d010.s003.t005","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"today%1:28:00::","call_ids":["semeval2013.d010.s003.t005__v1__a1","semeval2013.d010.s003.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"today%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043475,"input_uncached_usd":0.00003795,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.8459389170166105}
{"item_id":"semeval2013.d010.s003.t006","gold_sense_keys":["san_jose%1:15:00::"],"candidates":[{"index":1,"sense_key":"san_jose%1:15:01::","synset_id":"san_jose.n.01"},{"index":2,"sense_key":"san_jose%1:15:00::","synset_id":"san_jose.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"san_jose%1:15:00::","call_ids":["semeval2013.d010.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"san_jose%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048524999999999995,"input_uncached_usd":0.000018049999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.27594295796007}
{"item_id":"semeval2013.d010.s004.t001","gold_sense_keys":["number_one%1:24:00::"],"candidates":[{"index":1,"sense_key":"number_one%1:18:00::","synset_id":"number_one.n.01"},{"index":2,"sense_key":"number_one%1:24:00::","synset_id":"first.n.01"},{"index":3,"sense_key":"number_one%1:23:00::","synset_id":"first.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"number_one%1:23:00::","call_ids":["semeval2013.d010.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"number_one%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004626999999999999,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.370444041036535}
{"item_id":"semeval2013.d010.s004.t003","gold_sense_keys":["record%1:04:01::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"record%1:04:01::","call_ids":["semeval2013.d010.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"record%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037495,"input_uncached_usd":0.00003575,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.917729082982987}
{"item_id":"semeval2013.d010.s004.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d010.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":1937,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008108999999999999,"input_uncached_usd":0.000036099999999999997,"input_cached_usd":0.0,"output_usd":0.0007748,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.144165500008967}
{"item_id":"semeval2013.d010.s005.t000","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d010.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002783,"input_uncached_usd":0.0000415,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.146281042019837}
{"item_id":"semeval2013.d010.s005.t002","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":769,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048044999999999994,"input_uncached_usd":0.00003845,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.925395874946844}
{"item_id":"semeval2013.d010.s006.t001","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":3,"sense_key":"league%1:23:00::","synset_id":"league.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d010.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018199999999999998,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7073749999981374}
{"item_id":"semeval2013.d010.s006.t004","gold_sense_keys":["year%1:28:01::","year%1:28:02::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d010.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":546,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004433,"input_uncached_usd":0.0000273,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.133579250017647}
{"item_id":"semeval2013.d010.s006.t005","gold_sense_keys":["league%1:14:01::"],"candidates":[{"index":1,"sense_key":"league%1:14:01::","synset_id":"league.n.01"},{"index":2,"sense_key":"league%1:14:00::","synset_id":"league.n.02"},{"index":3,"sense_key":"league%1:23:00::","synset_id":"league.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"league%1:14:01::","call_ids":["semeval2013.d010.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"league%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023319999999999998,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.152533125015907}
{"item_id":"semeval2013.d010.s006.t007","gold_sense_keys":["competition%1:11:00::"],"candidates":[{"index":1,"sense_key":"competition%1:24:01::","synset_id":"competition.n.01"},{"index":2,"sense_key":"competition%1:11:00::","synset_id":"contest.n.01"},{"index":3,"sense_key":"competition%1:04:00::","synset_id":"competition.n.03"},{"index":4,"sense_key":"competition%1:18:00::","synset_id":"rival.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"competition%1:04:00::","call_ids":["semeval2013.d010.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"competition%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026294999999999997,"input_uncached_usd":0.00002615,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.244602833001409}
{"item_id":"semeval2013.d010.s007.t000","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d010.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006194000000000001,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.503725624992512}
{"item_id":"semeval2013.d010.s008.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031645,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3912927500205114}
{"item_id":"semeval2013.d010.s008.t001","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"experience%1:09:01::","call_ids":["semeval2013.d010.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"experience%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028835,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.707479832984973}
{"item_id":"semeval2013.d010.s008.t002","gold_sense_keys":["area%1:15:01::"],"candidates":[{"index":1,"sense_key":"area%1:15:01::","synset_id":"area.n.01"},{"index":2,"sense_key":"area%1:09:00::","synset_id":"area.n.02"},{"index":3,"sense_key":"area%1:08:00::","synset_id":"area.n.03"},{"index":4,"sense_key":"area%1:26:00::","synset_id":"sphere.n.01"},{"index":5,"sense_key":"area%1:06:00::","synset_id":"area.n.05"},{"index":6,"sense_key":"area%1:07:00::","synset_id":"area.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"area%1:15:01::","call_ids":["semeval2013.d010.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"area%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003479,"input_uncached_usd":0.0000339,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.300603541021701}
{"item_id":"semeval2013.d010.s008.t003","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"level%1:26:01::","call_ids":["semeval2013.d010.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"level%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042845,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.460828875016887}
{"item_id":"semeval2013.d010.s009.t000","gold_sense_keys":["peak%1:26:00::"],"candidates":[{"index":1,"sense_key":"peak%1:23:00::","synset_id":"extremum.n.02"},{"index":2,"sense_key":"peak%1:28:00::","synset_id":"flower.n.03"},{"index":3,"sense_key":"peak%1:26:00::","synset_id":"acme.n.01"},{"index":4,"sense_key":"peak%1:15:00::","synset_id":"peak.n.04"},{"index":5,"sense_key":"peak%1:25:01::","synset_id":"point.n.08"},{"index":6,"sense_key":"peak%1:15:01::","synset_id":"vertex.n.02"},{"index":7,"sense_key":"peak%1:06:00::","synset_id":"bill.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"peak%1:26:00::","call_ids":["semeval2013.d010.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"peak%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":732,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004014,"input_uncached_usd":0.000036599999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.94058195798425}
{"item_id":"semeval2013.d010.s009.t002","gold_sense_keys":["goalkeeper%1:18:00::"],"candidates":[{"index":1,"sense_key":"goalkeeper%1:18:00::","synset_id":"goalkeeper.n.01"},{"index":2,"sense_key":"goalkeeper%1:04:00::","synset_id":"goalkeeper.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"goalkeeper%1:18:00::","call_ids":["semeval2013.d010.s009.t002__v1__a1","semeval2013.d010.s009.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"goalkeeper%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1025,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007296499999999999,"input_uncached_usd":0.00005125,"input_cached_usd":0.0,"output_usd":0.0006784,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.502777707995847}
{"item_id":"semeval2013.d010.s009.t005","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d010.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043085,"input_uncached_usd":0.00004045,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.937011333997361}
{"item_id":"semeval2013.d010.s009.t006","gold_sense_keys":["london%1:15:00::"],"candidates":[{"index":1,"sense_key":"london%1:15:00::","synset_id":"london.n.01"},{"index":2,"sense_key":"london%1:18:00::","synset_id":"london.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"london%1:15:00::","call_ids":["semeval2013.d010.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"london%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018434999999999998,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.910517832962796}
{"item_id":"semeval2013.d010.s009.t007","gold_sense_keys":["record%1:04:00::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"record%1:04:00::","call_ids":["semeval2013.d010.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"record%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005073,"input_uncached_usd":0.0000401,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.392730541992933}
{"item_id":"semeval2013.d010.s009.t008","gold_sense_keys":["goalkeeper%1:18:00::"],"candidates":[{"index":1,"sense_key":"goalkeeper%1:18:00::","synset_id":"goalkeeper.n.01"},{"index":2,"sense_key":"goalkeeper%1:04:00::","synset_id":"goalkeeper.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"goalkeeper%1:18:00::","call_ids":["semeval2013.d010.s009.t008__v1__a1","semeval2013.d010.s009.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"goalkeeper%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1025,"cached_input_tokens":0,"output_tokens":1569,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006788499999999999,"input_uncached_usd":0.00005125,"input_cached_usd":0.0,"output_usd":0.0006276,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.562669625971466}
{"item_id":"semeval2013.d010.s009.t009","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":717,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037504999999999993,"input_uncached_usd":0.00003585,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.116216375026852}
{"item_id":"semeval2013.d010.s010.t000","gold_sense_keys":["today%1:28:01::"],"candidates":[{"index":1,"sense_key":"today%1:28:00::","synset_id":"today.n.01"},{"index":2,"sense_key":"today%1:28:01::","synset_id":"today.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"today%1:28:01::","call_ids":["semeval2013.d010.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"today%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033509999999999996,"input_uncached_usd":0.000021499999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.747893541993108}
{"item_id":"semeval2013.d010.s010.t001","gold_sense_keys":["record%1:04:00::"],"candidates":[{"index":1,"sense_key":"record%1:10:03::","synset_id":"record.n.01"},{"index":2,"sense_key":"record%1:06:00::","synset_id":"phonograph_record.n.01"},{"index":3,"sense_key":"record%1:23:00::","synset_id":"record.n.03"},{"index":4,"sense_key":"record%1:04:01::","synset_id":"record.n.04"},{"index":5,"sense_key":"record%1:10:02::","synset_id":"record.n.05"},{"index":6,"sense_key":"record%1:04:00::","synset_id":"record.n.06"},{"index":7,"sense_key":"record%1:21:00::","synset_id":"record.n.07"},{"index":8,"sense_key":"record%1:10:00::","synset_id":"criminal_record.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"record%1:04:00::","call_ids":["semeval2013.d010.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"record%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":768,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.000352,"input_uncached_usd":0.0000384,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.938148833985906}
{"item_id":"semeval2013.d010.s010.t002","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004246,"input_uncached_usd":0.0000342,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.190246667014435}
{"item_id":"semeval2013.d010.s011.t000","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":543,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031515,"input_uncached_usd":0.00002715,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3654375419719145}
{"item_id":"semeval2013.d010.s011.t001","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":703,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039995,"input_uncached_usd":0.00003515,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.350766125018708}
{"item_id":"semeval2013.d010.s012.t000","gold_sense_keys":["week%1:28:02::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2013.d010.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":453,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033624999999999996,"input_uncached_usd":0.000022649999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.855748084024526}
{"item_id":"semeval2013.d010.s012.t001","gold_sense_keys":["player%1:18:01::"],"candidates":[{"index":1,"sense_key":"player%1:18:01::","synset_id":"player.n.01"},{"index":2,"sense_key":"player%1:18:02::","synset_id":"musician.n.01"},{"index":3,"sense_key":"player%1:18:00::","synset_id":"actor.n.01"},{"index":4,"sense_key":"player%1:18:04::","synset_id":"player.n.04"},{"index":5,"sense_key":"player%1:18:03::","synset_id":"player.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"player%1:18:01::","call_ids":["semeval2013.d010.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"player%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023735,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.803999040974304}
{"item_id":"semeval2013.d010.s012.t002","gold_sense_keys":["team%1:14:00::"],"candidates":[{"index":1,"sense_key":"team%1:14:00::","synset_id":"team.n.01"},{"index":2,"sense_key":"team%1:14:01::","synset_id":"team.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"team%1:14:00::","call_ids":["semeval2013.d010.s012.t002__v1__a1","semeval2013.d010.s012.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"team%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00049025,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.000448,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.447950250003487}
{"item_id":"semeval2013.d010.s012.t003","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003218,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.519096207979601}
{"item_id":"semeval2013.d010.s012.t004","gold_sense_keys":["striker%1:18:02::"],"candidates":[{"index":1,"sense_key":"striker%1:18:02::","synset_id":"striker.n.01"},{"index":2,"sense_key":"striker%1:18:01::","synset_id":"striker.n.02"},{"index":3,"sense_key":"striker%1:18:00::","synset_id":"striker.n.03"},{"index":4,"sense_key":"striker%1:18:03::","synset_id":"hitter.n.02"},{"index":5,"sense_key":"striker%1:06:00::","synset_id":"striker.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"striker%1:18:02::","call_ids":["semeval2013.d010.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"striker%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023515,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.377946666034404}
{"item_id":"semeval2013.d010.s013.t000","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d010.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":864,"cached_input_tokens":0,"output_tokens":2640,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.0010991999999999998,"input_uncached_usd":0.0000432,"input_cached_usd":0.0,"output_usd":0.0010559999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.746207666990813}
{"item_id":"semeval2013.d010.s013.t001","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004305,"input_uncached_usd":0.0000401,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.223559750011191}
{"item_id":"semeval2013.d010.s015.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":886,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040909999999999997,"input_uncached_usd":0.0000443,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.748676499992143}
{"item_id":"semeval2013.d010.s015.t001","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016989999999999998,"input_uncached_usd":0.000035499999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9327623330173083}
{"item_id":"semeval2013.d010.s015.t002","gold_sense_keys":["striker%1:18:02::"],"candidates":[{"index":1,"sense_key":"striker%1:18:02::","synset_id":"striker.n.01"},{"index":2,"sense_key":"striker%1:18:01::","synset_id":"striker.n.02"},{"index":3,"sense_key":"striker%1:18:00::","synset_id":"striker.n.03"},{"index":4,"sense_key":"striker%1:18:03::","synset_id":"hitter.n.02"},{"index":5,"sense_key":"striker%1:06:00::","synset_id":"striker.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"striker%1:18:02::","call_ids":["semeval2013.d010.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"striker%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021124999999999998,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6739661669707857}
{"item_id":"semeval2013.d010.s016.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"game%1:04:00::","call_ids":["semeval2013.d010.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"game%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00078785,"input_uncached_usd":0.00003905,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.883794541005045}
{"item_id":"semeval2013.d010.s018.t000","gold_sense_keys":["match%1:11:00::"],"candidates":[{"index":1,"sense_key":"match%1:06:00::","synset_id":"match.n.01"},{"index":2,"sense_key":"match%1:11:00::","synset_id":"match.n.02"},{"index":3,"sense_key":"match%1:06:02::","synset_id":"match.n.03"},{"index":4,"sense_key":"match%1:06:01::","synset_id":"match.n.04"},{"index":5,"sense_key":"match%1:23:00::","synset_id":"match.n.05"},{"index":6,"sense_key":"match%1:18:01::","synset_id":"catch.n.03"},{"index":7,"sense_key":"match%1:18:00::","synset_id":"peer.n.01"},{"index":8,"sense_key":"match%1:14:00::","synset_id":"couple.n.02"},{"index":9,"sense_key":"match%1:09:00::","synset_id":"match.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"match%1:11:00::","call_ids":["semeval2013.d010.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"match%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":722,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004265,"input_uncached_usd":0.000036099999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.401941958989482}
{"item_id":"semeval2013.d010.s019.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d010.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030685,"input_uncached_usd":0.000044449999999999996,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.023289582983125}
{"item_id":"semeval2013.d010.s019.t001","gold_sense_keys":["presence%1:04:00::"],"candidates":[{"index":1,"sense_key":"presence%1:26:00::","synset_id":"presence.n.01"},{"index":2,"sense_key":"presence%1:15:00::","synset_id":"presence.n.02"},{"index":3,"sense_key":"presence%1:18:00::","synset_id":"presence.n.03"},{"index":4,"sense_key":"presence%1:09:00::","synset_id":"presence.n.04"},{"index":5,"sense_key":"presence%1:07:00::","synset_id":"bearing.n.03"},{"index":6,"sense_key":"presence%1:04:00::","synset_id":"presence.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"presence%1:04:00::","call_ids":["semeval2013.d010.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"presence%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054705,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.768299207964446}
{"item_id":"semeval2013.d010.s020.t000","gold_sense_keys":["rest%1:24:00::"],"candidates":[{"index":1,"sense_key":"rest%1:24:00::","synset_id":"remainder.n.01"},{"index":2,"sense_key":"rest%1:04:00::","synset_id":"rest.n.02"},{"index":3,"sense_key":"rest%1:28:00::","synset_id":"respite.n.04"},{"index":4,"sense_key":"rest%1:26:00::","synset_id":"rest.n.04"},{"index":5,"sense_key":"rest%1:26:01::","synset_id":"rest.n.05"},{"index":6,"sense_key":"rest%1:06:00::","synset_id":"rest.n.06"},{"index":7,"sense_key":"rest%1:10:00::","synset_id":"rest.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"rest%1:24:00::","call_ids":["semeval2013.d010.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"rest%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":673,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00078245,"input_uncached_usd":0.00003365,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.189884916006122}
{"item_id":"semeval2013.d010.s021.t000","gold_sense_keys":["game%1:04:03::"],"candidates":[{"index":1,"sense_key":"game%1:04:00::","synset_id":"game.n.01"},{"index":2,"sense_key":"game%1:04:03::","synset_id":"game.n.02"},{"index":3,"sense_key":"game%1:04:01::","synset_id":"game.n.03"},{"index":4,"sense_key":"game%1:05:00::","synset_id":"game.n.04"},{"index":5,"sense_key":"game%1:28:00::","synset_id":"game.n.05"},{"index":6,"sense_key":"game%1:23:00::","synset_id":"game.n.06"},{"index":7,"sense_key":"game%1:13:00::","synset_id":"game.n.07"},{"index":8,"sense_key":"game%1:09:00::","synset_id":"plot.n.01"},{"index":9,"sense_key":"game%1:06:00::","synset_id":"game.n.09"},{"index":10,"sense_key":"game%1:04:02::","synset_id":"game.n.10"},{"index":11,"sense_key":"game%1:04:04::","synset_id":"game.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"game%1:04:03::","call_ids":["semeval2013.d010.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"game%1:04:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004292,"input_uncached_usd":0.0000388,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.276445208990481}
{"item_id":"semeval2013.d010.s022.t000","gold_sense_keys":["scorer%1:18:00::"],"candidates":[{"index":1,"sense_key":"scorer%1:18:00::","synset_id":"scorer.n.01"},{"index":2,"sense_key":"scorer%1:18:01::","synset_id":"scorer.n.02"},{"index":3,"sense_key":"scorer%1:18:02::","synset_id":"scorekeeper.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"scorer%1:18:00::","call_ids":["semeval2013.d010.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"scorer%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012979999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7883019580040127}
{"item_id":"semeval2013.d010.s022.t001","gold_sense_keys":["goal%1:04:00::"],"candidates":[{"index":1,"sense_key":"goal%1:09:00::","synset_id":"goal.n.01"},{"index":2,"sense_key":"goal%1:15:00::","synset_id":"finish.n.04"},{"index":3,"sense_key":"goal%1:06:00::","synset_id":"goal.n.03"},{"index":4,"sense_key":"goal%1:04:00::","synset_id":"goal.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"goal%1:04:00::","call_ids":["semeval2013.d010.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"goal%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002115,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.111849625012837}
{"item_id":"semeval2013.d010.s023.t000","gold_sense_keys":["point%1:23:01::"],"candidates":[{"index":1,"sense_key":"point%1:09:00::","synset_id":"point.n.01"},{"index":2,"sense_key":"point%1:15:00::","synset_id":"point.n.02"},{"index":3,"sense_key":"point%1:10:01::","synset_id":"point.n.03"},{"index":4,"sense_key":"point%1:09:01::","synset_id":"detail.n.01"},{"index":5,"sense_key":"point%1:26:00::","synset_id":"degree.n.02"},{"index":6,"sense_key":"point%1:28:00::","synset_id":"point.n.06"},{"index":7,"sense_key":"point%1:09:02::","synset_id":"point.n.07"},{"index":8,"sense_key":"point%1:25:00::","synset_id":"point.n.08"},{"index":9,"sense_key":"point%1:25:02::","synset_id":"point.n.09"},{"index":10,"sense_key":"point%1:23:01::","synset_id":"point.n.10"},{"index":11,"sense_key":"point%1:17:02::","synset_id":"point.n.11"},{"index":12,"sense_key":"point%1:10:03::","synset_id":"item.n.01"},{"index":13,"sense_key":"point%1:24:04::","synset_id":"point.n.13"},{"index":14,"sense_key":"point%1:07:01::","synset_id":"point.n.14"},{"index":15,"sense_key":"point%1:06:00::","synset_id":"point.n.15"},{"index":16,"sense_key":"point%1:24:00::","synset_id":"compass_point.n.01"},{"index":17,"sense_key":"point%1:23:00::","synset_id":"point.n.17"},{"index":18,"sense_key":"point%1:21:00::","synset_id":"point.n.18"},{"index":19,"sense_key":"point%1:10:00::","synset_id":"period.n.07"},{"index":20,"sense_key":"point%1:10:02::","synset_id":"point.n.20"},{"index":21,"sense_key":"point%1:10:04::","synset_id":"decimal_point.n.01"},{"index":22,"sense_key":"point%1:07:00::","synset_id":"point.n.22"},{"index":23,"sense_key":"point%1:07:02::","synset_id":"point.n.23"},{"index":24,"sense_key":"point%1:06:02::","synset_id":"point.n.24"},{"index":25,"sense_key":"point%1:06:01::","synset_id":"point.n.25"},{"index":26,"sense_key":"point%1:06:03::","synset_id":"distributor_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"point%1:23:01::","call_ids":["semeval2013.d010.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"point%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1390,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005879,"input_uncached_usd":0.0000695,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5543779589934275}
{"item_id":"semeval2013.d011.s000.t001","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017835,"input_uncached_usd":0.00001795,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0622102500055917}
{"item_id":"semeval2013.d011.s000.t003","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d011.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":366,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020389999999999997,"input_uncached_usd":0.000018299999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.287539499986451}
{"item_id":"semeval2013.d011.s001.t000","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"},{"index":2,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"immigration%1:14:00::","call_ids":["semeval2013.d011.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"immigration%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006632,"input_uncached_usd":0.0000168,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.016873167012818}
{"item_id":"semeval2013.d011.s001.t001","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023035,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.254767332982738}
{"item_id":"semeval2013.d011.s001.t002","gold_sense_keys":["aspect%1:09:00::"],"candidates":[{"index":1,"sense_key":"aspect%1:09:00::","synset_id":"aspect.n.01"},{"index":2,"sense_key":"aspect%1:07:02::","synset_id":"aspect.n.02"},{"index":3,"sense_key":"aspect%1:09:01::","synset_id":"view.n.02"},{"index":4,"sense_key":"aspect%1:24:00::","synset_id":"aspect.n.04"},{"index":5,"sense_key":"aspect%1:07:01::","synset_id":"expression.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"aspect%1:09:00::","call_ids":["semeval2013.d011.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"aspect%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028425,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.411204333999194}
{"item_id":"semeval2013.d011.s001.t003","gold_sense_keys":["congestion%1:26:01::"],"candidates":[{"index":1,"sense_key":"congestion%1:26:00::","synset_id":"congestion.n.01"},{"index":2,"sense_key":"congestion%1:26:01::","synset_id":"congestion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"congestion%1:26:01::","call_ids":["semeval2013.d011.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"congestion%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":320,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033,"input_uncached_usd":0.000016,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.471193875011522}
{"item_id":"semeval2013.d011.s001.t006","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046074999999999995,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.388306082983036}
{"item_id":"semeval2013.d011.s001.t007","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d011.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005041,"input_uncached_usd":0.000036899999999999996,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.414976708008908}
{"item_id":"semeval2013.d011.s001.t008","gold_sense_keys":["compatibility%1:07:00::"],"candidates":[{"index":1,"sense_key":"compatibility%1:12:00::","synset_id":"compatibility.n.01"},{"index":2,"sense_key":"compatibility%1:07:00::","synset_id":"compatibility.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"compatibility%1:07:00::","call_ids":["semeval2013.d011.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"compatibility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038009999999999997,"input_uncached_usd":0.0000153,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.781257792026736}
{"item_id":"semeval2013.d011.s001.t009","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2013.d011.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00069115,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.30484554200666}
{"item_id":"semeval2013.d011.s001.t010","gold_sense_keys":["status%1:26:00::"],"candidates":[{"index":1,"sense_key":"status%1:26:00::","synset_id":"status.n.01"},{"index":2,"sense_key":"status%1:26:01::","synset_id":"condition.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"status%1:26:00::","call_ids":["semeval2013.d011.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"status%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043345,"input_uncached_usd":0.00001745,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.005702333000954}
{"item_id":"semeval2013.d011.s001.t011","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003131,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.297782332985662}
{"item_id":"semeval2013.d011.s001.t012","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004155,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.611884624988306}
{"item_id":"semeval2013.d011.s002.t000","gold_sense_keys":["harassment%1:04:00::"],"candidates":[{"index":1,"sense_key":"harassment%1:12:00::","synset_id":"harassment.n.01"},{"index":2,"sense_key":"harassment%1:04:00::","synset_id":"harassment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"harassment%1:04:00::","call_ids":["semeval2013.d011.s002.t000__v1__a1","semeval2013.d011.s002.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"harassment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058745,"input_uncached_usd":0.00003704999999999999,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.588959043030627}
{"item_id":"semeval2013.d011.s002.t001","gold_sense_keys":["student%1:18:00::"],"candidates":[{"index":1,"sense_key":"student%1:18:00::","synset_id":"student.n.01"},{"index":2,"sense_key":"student%1:18:01::","synset_id":"scholar.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"student%1:18:00::","call_ids":["semeval2013.d011.s002.t001__v1__a1","semeval2013.d011.s002.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"student%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048474999999999994,"input_uncached_usd":0.00003675,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.893432916025631}
{"item_id":"semeval2013.d011.s002.t002","gold_sense_keys":["france%1:15:00::"],"candidates":[{"index":1,"sense_key":"france%1:15:00::","synset_id":"france.n.01"},{"index":2,"sense_key":"france%1:18:00::","synset_id":"france.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"france%1:15:00::","call_ids":["semeval2013.d011.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"france%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":347,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015174999999999998,"input_uncached_usd":0.000017349999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9967966669937596}
{"item_id":"semeval2013.d011.s002.t003","gold_sense_keys":["talent%1:18:00::"],"candidates":[{"index":1,"sense_key":"talent%1:09:00::","synset_id":"endowment.n.01"},{"index":2,"sense_key":"talent%1:18:00::","synset_id":"talent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"talent%1:09:00::","call_ids":["semeval2013.d011.s002.t003__v1__a1","semeval2013.d011.s002.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"talent%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":2912,"reasoning_output_tokens":2880},"cost":{"currency":"USD","total_usd":0.00119965,"input_uncached_usd":0.00003484999999999999,"input_cached_usd":0.0,"output_usd":0.0011648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":18.751138458028436}
{"item_id":"semeval2013.d011.s002.t005","gold_sense_keys":["waiter%1:18:00::"],"candidates":[{"index":1,"sense_key":"waiter%1:18:00::","synset_id":"waiter.n.01"},{"index":2,"sense_key":"waiter%1:18:01::","synset_id":"waiter.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"waiter%1:18:00::","call_ids":["semeval2013.d011.s002.t005__v1__a1","semeval2013.d011.s002.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"waiter%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045715,"input_uncached_usd":0.000034750000000000004,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.624955957988277}
{"item_id":"semeval2013.d011.s003.t000","gold_sense_keys":["viewpoint%1:09:00::"],"candidates":[{"index":1,"sense_key":"viewpoint%1:09:00::","synset_id":"point_of_view.n.01"},{"index":2,"sense_key":"viewpoint%1:15:00::","synset_id":"vantage_point.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"viewpoint%1:09:00::","call_ids":["semeval2013.d011.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"viewpoint%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030715,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.359098249988165}
{"item_id":"semeval2013.d011.s004.t000","gold_sense_keys":["lot%1:23:00::"],"candidates":[{"index":1,"sense_key":"lot%1:23:00::","synset_id":"batch.n.02"},{"index":2,"sense_key":"lot%1:15:00::","synset_id":"lot.n.02"},{"index":3,"sense_key":"lot%1:14:00::","synset_id":"set.n.05"},{"index":4,"sense_key":"lot%1:26:00::","synset_id":"fortune.n.04"},{"index":5,"sense_key":"lot%1:06:00::","synset_id":"draw.n.04"},{"index":6,"sense_key":"lot%1:14:01::","synset_id":"bunch.n.03"},{"index":7,"sense_key":"lot%1:18:00::","synset_id":"lot.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lot%1:23:00::","call_ids":["semeval2013.d011.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lot%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":679,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029634999999999997,"input_uncached_usd":0.00003395,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.054887708975002}
{"item_id":"semeval2013.d011.s004.t002","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046974999999999995,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.715862207987811}
{"item_id":"semeval2013.d011.s004.t003","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022234999999999998,"input_uncached_usd":0.00003675,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.606624083011411}
{"item_id":"semeval2013.d011.s004.t004","gold_sense_keys":["subject%1:09:01::","subject%1:10:00::"],"candidates":[{"index":1,"sense_key":"subject%1:10:00::","synset_id":"subject.n.01"},{"index":2,"sense_key":"subject%1:06:00::","synset_id":"subject.n.02"},{"index":3,"sense_key":"subject%1:09:00::","synset_id":"discipline.n.01"},{"index":4,"sense_key":"subject%1:09:01::","synset_id":"topic.n.02"},{"index":5,"sense_key":"subject%1:10:01::","synset_id":"subject.n.05"},{"index":6,"sense_key":"subject%1:18:01::","synset_id":"subject.n.06"},{"index":7,"sense_key":"subject%1:18:00::","synset_id":"national.n.01"},{"index":8,"sense_key":"subject%1:10:02::","synset_id":"subject.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subject%1:10:00::","call_ids":["semeval2013.d011.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subject%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029884999999999997,"input_uncached_usd":0.00003645,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.83477270801086}
{"item_id":"semeval2013.d011.s005.t000","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":743,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00012034999999999998,"input_uncached_usd":0.000037149999999999995,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3354138339636847}
{"item_id":"semeval2013.d011.s005.t001","gold_sense_keys":["tone%1:07:03::"],"candidates":[{"index":1,"sense_key":"tone%1:10:01::","synset_id":"tone.n.01"},{"index":2,"sense_key":"tone%1:07:02::","synset_id":"tone.n.02"},{"index":3,"sense_key":"tone%1:07:01::","synset_id":"timbre.n.01"},{"index":4,"sense_key":"tone%1:26:01::","synset_id":"spirit.n.02"},{"index":5,"sense_key":"tone%1:07:00::","synset_id":"shade.n.02"},{"index":6,"sense_key":"tone%1:10:02::","synset_id":"note.n.03"},{"index":7,"sense_key":"tone%1:09:00::","synset_id":"tone.n.07"},{"index":8,"sense_key":"tone%1:26:02::","synset_id":"tonicity.n.01"},{"index":9,"sense_key":"tone%1:10:00::","synset_id":"tone.n.09"},{"index":10,"sense_key":"tone%1:07:03::","synset_id":"tone.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tone%1:26:01::","call_ids":["semeval2013.d011.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tone%1:26:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":885,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006394500000000001,"input_uncached_usd":0.00004425,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.555155624984764}
{"item_id":"semeval2013.d011.s005.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2013.d011.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056775,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.817222582991235}
{"item_id":"semeval2013.d011.s005.t004","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059975,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.5663717079442}
{"item_id":"semeval2013.d011.s005.t005","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2013.d011.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033959999999999996,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3718280420289375}
{"item_id":"semeval2013.d011.s005.t006","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":572,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018859999999999998,"input_uncached_usd":0.000028599999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7051654999959283}
{"item_id":"semeval2013.d011.s005.t007","gold_sense_keys":["france%1:15:00::"],"candidates":[{"index":1,"sense_key":"france%1:15:00::","synset_id":"france.n.01"},{"index":2,"sense_key":"france%1:18:00::","synset_id":"france.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"france%1:15:00::","call_ids":["semeval2013.d011.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"france%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038665,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.964267874951474}
{"item_id":"semeval2013.d011.s005.t009","gold_sense_keys":["misery%1:26:00::"],"candidates":[{"index":1,"sense_key":"misery%1:26:00::","synset_id":"misery.n.01"},{"index":2,"sense_key":"misery%1:12:00::","synset_id":"misery.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"misery%1:26:00::","call_ids":["semeval2013.d011.s005.t009__v1__a1","semeval2013.d011.s005.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"misery%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041785,"input_uncached_usd":0.000046649999999999996,"input_cached_usd":0.0,"output_usd":0.00037119999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.41482087498298}
{"item_id":"semeval2013.d011.s006.t000","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"difference%1:07:00::","call_ids":["semeval2013.d011.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"difference%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024024999999999999,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.167090790986549}
{"item_id":"semeval2013.d011.s006.t001","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"degree%1:07:00::","call_ids":["semeval2013.d011.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027065,"input_uncached_usd":0.000033849999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.722605083021335}
{"item_id":"semeval2013.d011.s006.t002","gold_sense_keys":["generosity%1:07:00::"],"candidates":[{"index":1,"sense_key":"generosity%1:07:00::","synset_id":"generosity.n.01"},{"index":2,"sense_key":"generosity%1:04:00::","synset_id":"generosity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"generosity%1:04:00::","call_ids":["semeval2013.d011.s006.t002__v1__a1","semeval2013.d011.s006.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"generosity%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":2592,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.00108275,"input_uncached_usd":0.00004594999999999999,"input_cached_usd":0.0,"output_usd":0.0010368,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.137847916048486}
{"item_id":"semeval2013.d011.s007.t000","gold_sense_keys":["fear%1:12:01::"],"candidates":[{"index":1,"sense_key":"fear%1:12:00::","synset_id":"fear.n.01"},{"index":2,"sense_key":"fear%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"fear%1:12:02::","synset_id":"fear.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fear%1:12:00::","call_ids":["semeval2013.d011.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fear%1:12:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013289999999999998,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.562266458000522}
{"item_id":"semeval2013.d011.s007.t001","gold_sense_keys":["invasion%1:11:00::"],"candidates":[{"index":1,"sense_key":"invasion%1:04:00::","synset_id":"invasion.n.01"},{"index":2,"sense_key":"invasion%1:11:00::","synset_id":"invasion.n.02"},{"index":3,"sense_key":"invasion%1:11:01::","synset_id":"invasion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"invasion%1:11:00::","call_ids":["semeval2013.d011.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"invasion%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020894999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.577308624982834}
{"item_id":"semeval2013.d011.s007.t002","gold_sense_keys":["territory%1:15:01::"],"candidates":[{"index":1,"sense_key":"territory%1:15:00::","synset_id":"district.n.01"},{"index":2,"sense_key":"territory%1:09:00::","synset_id":"territory.n.02"},{"index":3,"sense_key":"territory%1:15:01::","synset_id":"territory.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"territory%1:15:01::","call_ids":["semeval2013.d011.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"territory%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000803,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.6071217919816263}
{"item_id":"semeval2013.d011.s007.t003","gold_sense_keys":["horde%1:14:00::"],"candidates":[{"index":1,"sense_key":"horde%1:14:00::","synset_id":"horde.n.01"},{"index":2,"sense_key":"horde%1:14:02::","synset_id":"horde.n.02"},{"index":3,"sense_key":"horde%1:14:01::","synset_id":"drove.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"horde%1:14:00::","call_ids":["semeval2013.d011.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"horde%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033424999999999997,"input_uncached_usd":0.00002065,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.601653458958026}
{"item_id":"semeval2013.d011.s007.t005","gold_sense_keys":["left%1:14:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"left%1:14:00::","call_ids":["semeval2013.d011.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"left%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":558,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001623,"input_uncached_usd":0.000027899999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8136122089927085}
{"item_id":"semeval2013.d011.s007.t006","gold_sense_keys":["right%1:14:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"right%1:14:00::","call_ids":["semeval2013.d011.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"right%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":729,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024765,"input_uncached_usd":0.00003645,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.407827667018864}
{"item_id":"semeval2013.d011.s008.t000","gold_sense_keys":["side%1:14:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"side%1:14:00::","call_ids":["semeval2013.d011.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"side%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":822,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005595,"input_uncached_usd":0.000041099999999999996,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.297064540965948}
{"item_id":"semeval2013.d011.s008.t002","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"burden%1:09:01::","call_ids":["semeval2013.d011.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"burden%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051645,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.628384208015632}
{"item_id":"semeval2013.d011.s008.t003","gold_sense_keys":["economy%1:14:00::"],"candidates":[{"index":1,"sense_key":"economy%1:14:00::","synset_id":"economy.n.01"},{"index":2,"sense_key":"economy%1:09:01::","synset_id":"economy.n.02"},{"index":3,"sense_key":"economy%1:07:00::","synset_id":"economy.n.03"},{"index":4,"sense_key":"economy%1:04:00::","synset_id":"economy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economy%1:14:00::","call_ids":["semeval2013.d011.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economy%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":473,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018365,"input_uncached_usd":0.00002365,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.985927040979732}
{"item_id":"semeval2013.d011.s008.t004","gold_sense_keys":["society%1:14:00::"],"candidates":[{"index":1,"sense_key":"society%1:14:00::","synset_id":"society.n.01"},{"index":2,"sense_key":"society%1:14:01::","synset_id":"club.n.02"},{"index":3,"sense_key":"society%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"society%1:14:02::","synset_id":"society.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"society%1:14:00::","call_ids":["semeval2013.d011.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"society%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020954999999999997,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6960316249751486}
{"item_id":"semeval2013.d011.s009.t000","gold_sense_keys":["difference%1:07:00::"],"candidates":[{"index":1,"sense_key":"difference%1:07:00::","synset_id":"difference.n.01"},{"index":2,"sense_key":"difference%1:11:00::","synset_id":"deviation.n.01"},{"index":3,"sense_key":"difference%1:10:00::","synset_id":"dispute.n.01"},{"index":4,"sense_key":"difference%1:24:00::","synset_id":"difference.n.04"},{"index":5,"sense_key":"difference%1:23:00::","synset_id":"remainder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"difference%1:11:00::","call_ids":["semeval2013.d011.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"difference%1:11:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023799999999999998,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.581440917041618}
{"item_id":"semeval2013.d011.s009.t001","gold_sense_keys":["degree%1:07:00::","degree%1:26:01::"],"candidates":[{"index":1,"sense_key":"degree%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"degree%1:26:01::","synset_id":"degree.n.02"},{"index":3,"sense_key":"degree%1:10:00::","synset_id":"academic_degree.n.01"},{"index":4,"sense_key":"degree%1:23:00::","synset_id":"degree.n.04"},{"index":5,"sense_key":"degree%1:09:00::","synset_id":"degree.n.05"},{"index":6,"sense_key":"degree%1:23:03::","synset_id":"degree.n.06"},{"index":7,"sense_key":"degree%1:07:01::","synset_id":"degree.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"degree%1:07:00::","call_ids":["semeval2013.d011.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"degree%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019155,"input_uncached_usd":0.00003155,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.635957916965708}
{"item_id":"semeval2013.d011.s009.t002","gold_sense_keys":["generosity%1:07:00::"],"candidates":[{"index":1,"sense_key":"generosity%1:07:00::","synset_id":"generosity.n.01"},{"index":2,"sense_key":"generosity%1:04:00::","synset_id":"generosity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generosity%1:07:00::","call_ids":["semeval2013.d011.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generosity%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007682499999999999,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.63954837503843}
{"item_id":"semeval2013.d011.s010.t000","gold_sense_keys":["program%1:10:04::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"program%1:10:04::","call_ids":["semeval2013.d011.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"program%1:10:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001451,"input_uncached_usd":0.0000363,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7332057090243325}
{"item_id":"semeval2013.d011.s010.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2013.d011.s010.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004388,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.511664458026644}
{"item_id":"semeval2013.d011.s011.t000","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"notion%1:09:02::","call_ids":["semeval2013.d011.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"notion%1:09:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046799999999999994,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.571845291997306}
{"item_id":"semeval2013.d011.s011.t002","gold_sense_keys":["burden%1:09:01::"],"candidates":[{"index":1,"sense_key":"burden%1:09:01::","synset_id":"burden.n.01"},{"index":2,"sense_key":"burden%1:06:01::","synset_id":"load.n.01"},{"index":3,"sense_key":"burden%1:10:00::","synset_id":"effect.n.04"},{"index":4,"sense_key":"burden%1:09:00::","synset_id":"burden.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"burden%1:10:00::","call_ids":["semeval2013.d011.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"burden%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":502,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003387,"input_uncached_usd":0.0000251,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.5709761250182055}
{"item_id":"semeval2013.d011.s011.t003","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"problem%1:26:00::","call_ids":["semeval2013.d011.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"problem%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003626999999999999,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.684148583037313}
{"item_id":"semeval2013.d011.s012.t001","gold_sense_keys":["book%1:10:00::"],"candidates":[{"index":1,"sense_key":"book%1:10:00::","synset_id":"book.n.01"},{"index":2,"sense_key":"book%1:06:00::","synset_id":"book.n.02"},{"index":3,"sense_key":"book%1:10:02::","synset_id":"record.n.05"},{"index":4,"sense_key":"book%1:10:01::","synset_id":"script.n.01"},{"index":5,"sense_key":"book%1:21:00::","synset_id":"ledger.n.01"},{"index":6,"sense_key":"book%1:14:01::","synset_id":"book.n.06"},{"index":7,"sense_key":"book%1:14:00::","synset_id":"book.n.07"},{"index":8,"sense_key":"book%1:10:05::","synset_id":"koran.n.01"},{"index":9,"sense_key":"book%1:10:04::","synset_id":"bible.n.01"},{"index":10,"sense_key":"book%1:10:03::","synset_id":"book.n.10"},{"index":11,"sense_key":"book%1:06:02::","synset_id":"book.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"book%1:10:00::","call_ids":["semeval2013.d011.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"book%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":836,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004322,"input_uncached_usd":0.0000418,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.253787832974922}
{"item_id":"semeval2013.d011.s012.t003","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"},{"index":2,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028425,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.286412000015844}
{"item_id":"semeval2013.d011.s012.t004","gold_sense_keys":["people%1:14:00::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"people%1:14:00::","call_ids":["semeval2013.d011.s012.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"people%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046814999999999997,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0822736250120215}
{"item_id":"semeval2013.d011.s012.t005","gold_sense_keys":["migration%1:04:00::"],"candidates":[{"index":1,"sense_key":"migration%1:04:00::","synset_id":"migration.n.01"},{"index":2,"sense_key":"migration%1:14:00::","synset_id":"migration.n.02"},{"index":3,"sense_key":"migration%1:11:01::","synset_id":"migration.n.03"},{"index":4,"sense_key":"migration%1:11:00::","synset_id":"migration.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"migration%1:04:00::","call_ids":["semeval2013.d011.s012.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"migration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00008195,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.6291800829931162}
{"item_id":"semeval2013.d011.s012.t007","gold_sense_keys":["future%1:28:00::"],"candidates":[{"index":1,"sense_key":"future%1:28:00::","synset_id":"future.n.01"},{"index":2,"sense_key":"future%1:10:00::","synset_id":"future.n.02"},{"index":3,"sense_key":"future%1:06:00::","synset_id":"future.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"future%1:28:00::","call_ids":["semeval2013.d011.s012.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"future%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001574,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.949125416984316}
{"item_id":"semeval2013.d011.s012.t009","gold_sense_keys":["specialist%1:18:00::"],"candidates":[{"index":1,"sense_key":"specialist%1:18:00::","synset_id":"specialist.n.01"},{"index":2,"sense_key":"specialist%1:18:01::","synset_id":"specialist.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"specialist%1:18:00::","call_ids":["semeval2013.d011.s012.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"specialist%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046274999999999995,"input_uncached_usd":0.00002115,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.074100999976508}
{"item_id":"semeval2013.d011.s012.t010","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d011.s012.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004672999999999999,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.210650166962296}
{"item_id":"semeval2013.d011.s012.t011","gold_sense_keys":["migration%1:04:00::"],"candidates":[{"index":1,"sense_key":"migration%1:04:00::","synset_id":"migration.n.01"},{"index":2,"sense_key":"migration%1:14:00::","synset_id":"migration.n.02"},{"index":3,"sense_key":"migration%1:11:01::","synset_id":"migration.n.03"},{"index":4,"sense_key":"migration%1:11:00::","synset_id":"migration.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"migration%1:04:00::","call_ids":["semeval2013.d011.s012.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"migration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015874999999999998,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.16837349999696}
{"item_id":"semeval2013.d011.s012.t012","gold_sense_keys":["flow%1:26:00::"],"candidates":[{"index":1,"sense_key":"flow%1:11:00::","synset_id":"flow.n.01"},{"index":2,"sense_key":"flow%1:28:00::","synset_id":"flow.n.02"},{"index":3,"sense_key":"flow%1:04:00::","synset_id":"flow.n.03"},{"index":4,"sense_key":"flow%1:22:00::","synset_id":"flow.n.04"},{"index":5,"sense_key":"flow%1:26:00::","synset_id":"stream.n.04"},{"index":6,"sense_key":"flow%1:14:00::","synset_id":"stream.n.02"},{"index":7,"sense_key":"flow%1:22:01::","synset_id":"menstruation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"flow%1:26:00::","call_ids":["semeval2013.d011.s012.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"flow%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":672,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027039999999999996,"input_uncached_usd":0.0000336,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.569599665992428}
{"item_id":"semeval2013.d011.s012.t015","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2013.d011.s012.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003144,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.678691917040851}
{"item_id":"semeval2013.d011.s012.t016","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s012.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003185,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.668337916024029}
{"item_id":"semeval2013.d011.s013.t000","gold_sense_keys":["immigration%1:04:00::"],"candidates":[{"index":1,"sense_key":"immigration%1:04:00::","synset_id":"immigration.n.01"},{"index":2,"sense_key":"immigration%1:14:00::","synset_id":"immigration.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"immigration%1:04:00::","call_ids":["semeval2013.d011.s013.t000__v1__a1","semeval2013.d011.s013.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"immigration%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":931,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00064815,"input_uncached_usd":0.00004654999999999999,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.351767709013075}
{"item_id":"semeval2013.d011.s013.t001","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["semeval2013.d011.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001158,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9544397909776308}
{"item_id":"semeval2013.d011.s013.t002","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2013.d011.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046859999999999995,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.312705375021324}
{"item_id":"semeval2013.d011.s014.t000","gold_sense_keys":["fear%1:12:01::"],"candidates":[{"index":1,"sense_key":"fear%1:12:00::","synset_id":"fear.n.01"},{"index":2,"sense_key":"fear%1:12:01::","synset_id":"concern.n.02"},{"index":3,"sense_key":"fear%1:12:02::","synset_id":"fear.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fear%1:12:01::","call_ids":["semeval2013.d011.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fear%1:12:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016035,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.867787541996222}
{"item_id":"semeval2013.d011.s014.t001","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2013.d011.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006479999999999999,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.9358583749854}
{"item_id":"semeval2013.d011.s014.t003","gold_sense_keys":["notion%1:09:02::"],"candidates":[{"index":1,"sense_key":"notion%1:09:02::","synset_id":"impression.n.01"},{"index":2,"sense_key":"notion%1:09:00::","synset_id":"notion.n.02"},{"index":3,"sense_key":"notion%1:09:01::","synset_id":"notion.n.03"},{"index":4,"sense_key":"notion%1:06:00::","synset_id":"notion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"notion%1:09:00::","call_ids":["semeval2013.d011.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"notion%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006229,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.326783958007582}
{"item_id":"semeval2013.d011.s014.t004","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2013.d011.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021265,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9340327079989947}
{"item_id":"semeval2013.d011.s014.t005","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d011.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004141,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.8615157079766504}
{"item_id":"semeval2013.d011.s014.t006","gold_sense_keys":["pressure%1:07:00::"],"candidates":[{"index":1,"sense_key":"pressure%1:19:00::","synset_id":"pressure.n.01"},{"index":2,"sense_key":"pressure%1:07:00::","synset_id":"pressure.n.02"},{"index":3,"sense_key":"pressure%1:04:00::","synset_id":"press.n.09"},{"index":4,"sense_key":"pressure%1:26:00::","synset_id":"imperativeness.n.01"},{"index":5,"sense_key":"pressure%1:09:00::","synset_id":"pressure.n.05"},{"index":6,"sense_key":"pressure%1:26:01::","synset_id":"pressure.n.06"},{"index":7,"sense_key":"pressure%1:19:01::","synset_id":"atmospheric_pressure.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pressure%1:07:00::","call_ids":["semeval2013.d011.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pressure%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":691,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045055,"input_uncached_usd":0.00003455,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.086984375026077}
{"item_id":"semeval2013.d011.s015.t000","gold_sense_keys":["experience%1:09:01::"],"candidates":[{"index":1,"sense_key":"experience%1:09:01::","synset_id":"experience.n.01"},{"index":2,"sense_key":"experience%1:09:00::","synset_id":"experience.n.02"},{"index":3,"sense_key":"experience%1:11:00::","synset_id":"experience.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"experience%1:09:00::","call_ids":["semeval2013.d011.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"experience%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039035,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.311512207961641}
{"item_id":"semeval2013.d011.s015.t003","gold_sense_keys":["replacement%1:09:00::","replacement%1:18:02::"],"candidates":[{"index":1,"sense_key":"replacement%1:04:00::","synset_id":"replacement.n.01"},{"index":2,"sense_key":"replacement%1:18:02::","synset_id":"surrogate.n.01"},{"index":3,"sense_key":"replacement%1:11:00::","synset_id":"substitution.n.01"},{"index":4,"sense_key":"replacement%1:09:00::","synset_id":"substitute.n.01"},{"index":5,"sense_key":"replacement%1:22:00::","synset_id":"refilling.n.01"},{"index":6,"sense_key":"replacement%1:18:01::","synset_id":"successor.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"replacement%1:18:02::","call_ids":["semeval2013.d011.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"replacement%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006010000000000001,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.596654957975261}
{"item_id":"semeval2013.d011.s015.t004","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"labor%1:04:00::","call_ids":["semeval2013.d011.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"labor%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005048499999999999,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.05375662498409}
{"item_id":"semeval2013.d011.s015.t005","gold_sense_keys":["trade%1:04:00::"],"candidates":[{"index":1,"sense_key":"trade%1:04:05::","synset_id":"trade.n.01"},{"index":2,"sense_key":"trade%1:04:00::","synset_id":"trade.n.02"},{"index":3,"sense_key":"trade%1:04:01::","synset_id":"trade.n.03"},{"index":4,"sense_key":"trade%1:04:02::","synset_id":"deal.n.01"},{"index":5,"sense_key":"trade%1:14:00::","synset_id":"craft.n.03"},{"index":6,"sense_key":"trade%1:19:00::","synset_id":"trade_wind.n.01"},{"index":7,"sense_key":"trade%1:04:03::","synset_id":"barter.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"trade%1:04:00::","call_ids":["semeval2013.d011.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"trade%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006822,"input_uncached_usd":0.000035799999999999996,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.255016208975576}
{"item_id":"semeval2013.d011.s015.t006","gold_sense_keys":["shortage%1:26:00::"],"candidates":[{"index":1,"sense_key":"shortage%1:07:00::","synset_id":"deficit.n.01"},{"index":2,"sense_key":"shortage%1:26:00::","synset_id":"dearth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shortage%1:07:00::","call_ids":["semeval2013.d011.s015.t006__v1__a1","semeval2013.d011.s015.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shortage%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1011,"cached_input_tokens":0,"output_tokens":2080,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008825500000000001,"input_uncached_usd":0.000050549999999999995,"input_cached_usd":0.0,"output_usd":0.000832,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.277503999008331}
{"item_id":"semeval2013.d011.s015.t007","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047965,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.946677082974929}
{"item_id":"semeval2013.d011.s016.t001","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d011.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057245,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.200019540963694}
{"item_id":"semeval2013.d011.s016.t002","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"labor%1:14:00::","call_ids":["semeval2013.d011.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"labor%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008111500000000001,"input_uncached_usd":0.00003675,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.688535458000842}
{"item_id":"semeval2013.d011.s016.t003","gold_sense_keys":["one%1:09:00::"],"candidates":[{"index":1,"sense_key":"one%1:23:00::","synset_id":"one.n.01"},{"index":2,"sense_key":"one%1:09:00::","synset_id":"one.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"one%1:09:00::","call_ids":["semeval2013.d011.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"one%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":471,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020915,"input_uncached_usd":0.00002355,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.179167583002709}
{"item_id":"semeval2013.d011.s016.t004","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2013.d011.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007005,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.048580417002086}
{"item_id":"semeval2013.d011.s016.t005","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2013.d011.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048945,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.12811812502332}
{"item_id":"semeval2013.d011.s016.t006","gold_sense_keys":["offer%1:10:00::"],"candidates":[{"index":1,"sense_key":"offer%1:10:01::","synset_id":"offer.n.01"},{"index":2,"sense_key":"offer%1:10:00::","synset_id":"offer.n.02"},{"index":3,"sense_key":"offer%1:04:01::","synset_id":"crack.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"offer%1:10:00::","call_ids":["semeval2013.d011.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"offer%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002105,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4768349999794737}
{"item_id":"semeval2013.d011.s016.t007","gold_sense_keys":["employment%1:04:00::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2013.d011.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":545,"cached_input_tokens":0,"output_tokens":2705,"reasoning_output_tokens":2688},"cost":{"currency":"USD","total_usd":0.0011092499999999998,"input_uncached_usd":0.000027249999999999998,"input_cached_usd":0.0,"output_usd":0.0010819999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.367918292002287}
{"item_id":"semeval2013.d011.s017.t000","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044895,"input_uncached_usd":0.00003295,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.905838290986139}
{"item_id":"semeval2013.d011.s018.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2013.d011.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":827,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045735,"input_uncached_usd":0.000041349999999999995,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.8440801670076326}
{"item_id":"semeval2013.d011.s018.t002","gold_sense_keys":["country%1:14:00::","country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2013.d011.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":605,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047184999999999995,"input_uncached_usd":0.00003025,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.741181917022914}
{"item_id":"semeval2013.d011.s018.t003","gold_sense_keys":["shortage%1:26:00::"],"candidates":[{"index":1,"sense_key":"shortage%1:07:00::","synset_id":"deficit.n.01"},{"index":2,"sense_key":"shortage%1:26:00::","synset_id":"dearth.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"shortage%1:07:00::","call_ids":["semeval2013.d011.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"shortage%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00079675,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.281422416039277}
{"item_id":"semeval2013.d011.s018.t005","gold_sense_keys":["complementarity%1:24:02::"],"candidates":[{"index":1,"sense_key":"complementarity%1:24:01::","synset_id":"complementarity.n.01"},{"index":2,"sense_key":"complementarity%1:24:02::","synset_id":"complementarity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"complementarity%1:24:02::","call_ids":["semeval2013.d011.s018.t005__v1__a1","semeval2013.d011.s018.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"complementarity%1:24:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":911,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00072395,"input_uncached_usd":0.000045549999999999996,"input_cached_usd":0.0,"output_usd":0.0006783999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.355451125011314}
{"item_id":"semeval2013.d011.s018.t007","gold_sense_keys":["arrival%1:04:01::"],"candidates":[{"index":1,"sense_key":"arrival%1:04:00::","synset_id":"arrival.n.01"},{"index":2,"sense_key":"arrival%1:04:01::","synset_id":"arrival.n.02"},{"index":3,"sense_key":"arrival%1:18:00::","synset_id":"arrival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"arrival%1:18:00::","call_ids":["semeval2013.d011.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"arrival%1:18:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020794999999999999,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.15999841602752}
{"item_id":"semeval2013.d011.s018.t009","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021644999999999998,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8141787499771453}
{"item_id":"semeval2013.d011.s019.t000","gold_sense_keys":["rise%1:11:00::"],"candidates":[{"index":1,"sense_key":"rise%1:11:00::","synset_id":"rise.n.01"},{"index":2,"sense_key":"rise%1:04:00::","synset_id":"rise.n.02"},{"index":3,"sense_key":"rise%1:17:00::","synset_id":"ascent.n.01"},{"index":4,"sense_key":"rise%1:11:02::","synset_id":"rise.n.04"},{"index":5,"sense_key":"rise%1:07:02::","synset_id":"raise.n.01"},{"index":6,"sense_key":"rise%1:07:01::","synset_id":"upgrade.n.04"},{"index":7,"sense_key":"rise%1:11:01::","synset_id":"lift.n.04"},{"index":8,"sense_key":"rise%1:11:03::","synset_id":"emanation.n.03"},{"index":9,"sense_key":"rise%1:07:03::","synset_id":"rise.n.09"},{"index":10,"sense_key":"rise%1:04:01::","synset_id":"advance.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"rise%1:07:02::","call_ids":["semeval2013.d011.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"rise%1:07:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003285,"input_uncached_usd":0.000040499999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.689321707992349}
{"item_id":"semeval2013.d011.s019.t001","gold_sense_keys":["flow%1:26:00::"],"candidates":[{"index":1,"sense_key":"flow%1:11:00::","synset_id":"flow.n.01"},{"index":2,"sense_key":"flow%1:28:00::","synset_id":"flow.n.02"},{"index":3,"sense_key":"flow%1:04:00::","synset_id":"flow.n.03"},{"index":4,"sense_key":"flow%1:22:00::","synset_id":"flow.n.04"},{"index":5,"sense_key":"flow%1:26:00::","synset_id":"stream.n.04"},{"index":6,"sense_key":"flow%1:14:00::","synset_id":"stream.n.02"},{"index":7,"sense_key":"flow%1:22:01::","synset_id":"menstruation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"flow%1:26:00::","call_ids":["semeval2013.d011.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"flow%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042635,"input_uncached_usd":0.00003595,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.502654124982655}
{"item_id":"semeval2013.d011.s019.t002","gold_sense_keys":["increase%1:07:00::","increase%1:11:00::"],"candidates":[{"index":1,"sense_key":"increase%1:23:00::","synset_id":"addition.n.03"},{"index":2,"sense_key":"increase%1:11:00::","synset_id":"increase.n.02"},{"index":3,"sense_key":"increase%1:22:00::","synset_id":"increase.n.03"},{"index":4,"sense_key":"increase%1:07:00::","synset_id":"increase.n.04"},{"index":5,"sense_key":"increase%1:04:00::","synset_id":"increase.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"increase%1:07:00::","call_ids":["semeval2013.d011.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"increase%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034415,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.495819624979049}
{"item_id":"semeval2013.d011.s020.t000","gold_sense_keys":["fact%1:09:01::"],"candidates":[{"index":1,"sense_key":"fact%1:09:01::","synset_id":"fact.n.01"},{"index":2,"sense_key":"fact%1:10:01::","synset_id":"fact.n.02"},{"index":3,"sense_key":"fact%1:26:01::","synset_id":"fact.n.03"},{"index":4,"sense_key":"fact%1:09:02::","synset_id":"fact.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fact%1:10:01::","call_ids":["semeval2013.d011.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fact%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":562,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039289999999999996,"input_uncached_usd":0.0000281,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.748921749996953}
{"item_id":"semeval2013.d011.s020.t001","gold_sense_keys":["diversity%1:07:02::"],"candidates":[{"index":1,"sense_key":"diversity%1:07:02::","synset_id":"diverseness.n.01"},{"index":2,"sense_key":"diversity%1:07:01::","synset_id":"diversity.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"diversity%1:07:02::","call_ids":["semeval2013.d011.s020.t001__v1__a1","semeval2013.d011.s020.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"diversity%1:07:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":985,"cached_input_tokens":0,"output_tokens":1504,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006508499999999999,"input_uncached_usd":0.00004925,"input_cached_usd":0.0,"output_usd":0.0006016,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.146768207952846}
{"item_id":"semeval2013.d011.s020.t003","gold_sense_keys":["creation%1:04:00::"],"candidates":[{"index":1,"sense_key":"creation%1:04:00::","synset_id":"creation.n.01"},{"index":2,"sense_key":"creation%1:06:00::","synset_id":"creation.n.02"},{"index":3,"sense_key":"creation%1:11:00::","synset_id":"creation.n.03"},{"index":4,"sense_key":"creation%1:04:01::","synset_id":"initiation.n.02"},{"index":5,"sense_key":"creation%1:04:02::","synset_id":"creation.n.05"},{"index":6,"sense_key":"creation%1:17:00::","synset_id":"universe.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"creation%1:04:00::","call_ids":["semeval2013.d011.s020.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"creation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031955,"input_uncached_usd":0.00003155,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.951039333012886}
{"item_id":"semeval2013.d011.s020.t004","gold_sense_keys":["idea%1:09:00::"],"candidates":[{"index":1,"sense_key":"idea%1:09:00::","synset_id":"idea.n.01"},{"index":2,"sense_key":"idea%1:09:01::","synset_id":"mind.n.06"},{"index":3,"sense_key":"idea%1:09:03::","synset_id":"idea.n.03"},{"index":4,"sense_key":"idea%1:09:02::","synset_id":"estimate.n.01"},{"index":5,"sense_key":"idea%1:10:00::","synset_id":"theme.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"idea%1:09:00::","call_ids":["semeval2013.d011.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"idea%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003452,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.3279192090267316}
{"item_id":"semeval2013.d011.s020.t006","gold_sense_keys":["proportion%1:24:01::"],"candidates":[{"index":1,"sense_key":"proportion%1:24:00::","synset_id":"proportion.n.01"},{"index":2,"sense_key":"proportion%1:07:00::","synset_id":"proportion.n.02"},{"index":3,"sense_key":"proportion%1:25:00::","synset_id":"symmetry.n.02"},{"index":4,"sense_key":"proportion%1:24:01::","synset_id":"proportion.n.04"},{"index":5,"sense_key":"proportion%1:07:01::","synset_id":"proportion.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"proportion%1:24:01::","call_ids":["semeval2013.d011.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"proportion%1:24:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024244999999999998,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.432848625001498}
{"item_id":"semeval2013.d011.s020.t008","gold_sense_keys":["american%3:01:01::"],"candidates":[{"index":1,"sense_key":"american%3:01:01::","synset_id":"american.a.01"},{"index":2,"sense_key":"american%3:01:00::","synset_id":"american.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"american%3:01:01::","call_ids":["semeval2013.d011.s020.t008__v1__a1","semeval2013.d011.s020.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"american%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":989,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047185,"input_uncached_usd":0.000049449999999999996,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.683287542022299}
{"item_id":"semeval2013.d011.s020.t010","gold_sense_keys":["winner%1:18:00::"],"candidates":[{"index":1,"sense_key":"winner%1:18:00::","synset_id":"winner.n.01"},{"index":2,"sense_key":"winner%1:18:01::","synset_id":"winner.n.02"},{"index":3,"sense_key":"winner%1:18:02::","synset_id":"achiever.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"winner%1:18:02::","call_ids":["semeval2013.d011.s020.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"winner%1:18:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":491,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021014999999999999,"input_uncached_usd":0.00002455,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.415137375006452}
{"item_id":"semeval2013.d011.s020.t012","gold_sense_keys":["yahoo%1:10:00::"],"candidates":[{"index":1,"sense_key":"yahoo%1:18:01::","synset_id":"yokel.n.01"},{"index":2,"sense_key":"yahoo%1:18:00::","synset_id":"yahoo.n.02"},{"index":3,"sense_key":"yahoo%1:10:00::","synset_id":"yahoo.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"yahoo%1:10:00::","call_ids":["semeval2013.d011.s020.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"yahoo%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001344,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5661855829530396}
{"item_id":"semeval2013.d011.s021.t001","gold_sense_keys":["contributor%1:18:00::"],"candidates":[{"index":1,"sense_key":"contributor%1:18:00::","synset_id":"subscriber.n.03"},{"index":2,"sense_key":"contributor%1:18:01::","synset_id":"contributor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contributor%1:18:00::","call_ids":["semeval2013.d011.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contributor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":472,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005420000000000001,"input_uncached_usd":0.000023599999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.653054416994564}
{"item_id":"semeval2013.d011.s021.t003","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"},{"index":2,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"budget%1:21:02::","call_ids":["semeval2013.d011.s021.t003__v1__a1","semeval2013.d011.s021.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1013,"cached_input_tokens":0,"output_tokens":1889,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0008062499999999999,"input_uncached_usd":0.00005065,"input_cached_usd":0.0,"output_usd":0.0007555999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.183760666986927}
{"item_id":"semeval2013.d011.s022.t001","gold_sense_keys":["labor%1:14:00::"],"candidates":[{"index":1,"sense_key":"labor%1:14:00::","synset_id":"labor.n.01"},{"index":2,"sense_key":"labor%1:04:00::","synset_id":"labor.n.02"},{"index":3,"sense_key":"labor%1:26:00::","synset_id":"parturiency.n.01"},{"index":4,"sense_key":"labor%1:14:04::","synset_id":"labor_movement.n.01"},{"index":5,"sense_key":"labor%1:14:02::","synset_id":"british_labour_party.n.01"},{"index":6,"sense_key":"labor%1:14:01::","synset_id":"department_of_labor.n.01"},{"index":7,"sense_key":"labor%1:04:01::","synset_id":"undertaking.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"labor%1:04:00::","call_ids":["semeval2013.d011.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"labor%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007356499999999999,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.497950791963376}
{"item_id":"semeval2013.d011.s022.t002","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":2,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":3,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":4,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":5,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":6,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2013.d011.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":766,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007615,"input_uncached_usd":0.000038299999999999996,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.4688774170354}
{"item_id":"semeval2013.d011.s022.t006","gold_sense_keys":["age%1:28:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2013.d011.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":611,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019055,"input_uncached_usd":0.00003055,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5344154999474995}
{"item_id":"semeval2013.d011.s022.t007","gold_sense_keys":["contribution%1:21:01::"],"candidates":[{"index":1,"sense_key":"contribution%1:04:02::","synset_id":"contribution.n.01"},{"index":2,"sense_key":"contribution%1:21:00::","synset_id":"contribution.n.02"},{"index":3,"sense_key":"contribution%1:04:00::","synset_id":"contribution.n.03"},{"index":4,"sense_key":"contribution%1:21:01::","synset_id":"contribution.n.04"},{"index":5,"sense_key":"contribution%1:04:01::","synset_id":"contribution.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"contribution%1:21:01::","call_ids":["semeval2013.d011.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"contribution%1:21:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006249000000000001,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.54590000002645}
{"item_id":"semeval2013.d011.s022.t009","gold_sense_keys":["expenditure%1:21:00::"],"candidates":[{"index":1,"sense_key":"expenditure%1:21:00::","synset_id":"outgo.n.01"},{"index":2,"sense_key":"expenditure%1:04:00::","synset_id":"expending.n.01"},{"index":3,"sense_key":"expenditure%1:04:01::","synset_id":"consumption.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"expenditure%1:21:00::","call_ids":["semeval2013.d011.s022.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"expenditure%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003124,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.054049083031714}
{"item_id":"semeval2013.d011.s022.t010","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:00::","synset_id":"average.n.01"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:01::","synset_id":"average.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"average%1:09:01::","call_ids":["semeval2013.d011.s022.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"average%1:09:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051765,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.281413167016581}
{"item_id":"semeval2013.d011.s022.t012","gold_sense_keys":["budget%1:21:02::"],"candidates":[{"index":1,"sense_key":"budget%1:21:02::","synset_id":"budget.n.01"},{"index":2,"sense_key":"budget%1:21:03::","synset_id":"budget.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"budget%1:21:02::","call_ids":["semeval2013.d011.s022.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"budget%1:21:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004139,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.775319207983557}
{"item_id":"semeval2013.d011.s023.t001","gold_sense_keys":["mandate%1:04:00::"],"candidates":[{"index":1,"sense_key":"mandate%1:10:00::","synset_id":"mandate.n.01"},{"index":2,"sense_key":"mandate%1:15:00::","synset_id":"mandate.n.02"},{"index":3,"sense_key":"mandate%1:04:00::","synset_id":"mandate.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mandate%1:04:00::","call_ids":["semeval2013.d011.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mandate%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010929999999999999,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.7572714999550954}
{"item_id":"semeval2013.d011.s023.t003","gold_sense_keys":["growth%1:22:02::"],"candidates":[{"index":1,"sense_key":"growth%1:22:00::","synset_id":"growth.n.01"},{"index":2,"sense_key":"growth%1:22:01::","synset_id":"growth.n.02"},{"index":3,"sense_key":"growth%1:22:02::","synset_id":"increase.n.03"},{"index":4,"sense_key":"growth%1:14:00::","synset_id":"growth.n.04"},{"index":5,"sense_key":"growth%1:11:01::","synset_id":"emergence.n.01"},{"index":6,"sense_key":"growth%1:26:00::","synset_id":"growth.n.06"},{"index":7,"sense_key":"growth%1:17:00::","synset_id":"growth.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"growth%1:22:02::","call_ids":["semeval2013.d011.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"growth%1:22:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":698,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001949,"input_uncached_usd":0.0000349,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1063763750134967}
{"item_id":"semeval2013.d011.s024.t000","gold_sense_keys":["question%1:10:01::"],"candidates":[{"index":1,"sense_key":"question%1:10:00::","synset_id":"question.n.01"},{"index":2,"sense_key":"question%1:10:01::","synset_id":"question.n.02"},{"index":3,"sense_key":"question%1:10:03::","synset_id":"question.n.03"},{"index":4,"sense_key":"question%1:07:00::","synset_id":"doubt.n.02"},{"index":5,"sense_key":"question%1:10:02::","synset_id":"motion.n.05"},{"index":6,"sense_key":"question%1:10:04::","synset_id":"question.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"question%1:10:01::","call_ids":["semeval2013.d011.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"question%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":649,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032045,"input_uncached_usd":0.000032449999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.894891541975085}
{"item_id":"semeval2013.d011.s024.t002","gold_sense_keys":["effect%1:19:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2013.d011.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016524999999999998,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.798074542020913}
{"item_id":"semeval2013.d012.s000.t000","gold_sense_keys":["pound%1:23:01::"],"candidates":[{"index":1,"sense_key":"pound%1:23:09::","synset_id":"pound.n.01"},{"index":2,"sense_key":"pound%1:23:01::","synset_id":"british_pound.n.01"},{"index":3,"sense_key":"pound%1:23:10::","synset_id":"pound.n.03"},{"index":4,"sense_key":"pound%1:23:08::","synset_id":"syrian_pound.n.01"},{"index":5,"sense_key":"pound%1:23:07::","synset_id":"sudanese_pound.n.01"},{"index":6,"sense_key":"pound%1:23:05::","synset_id":"lebanese_pound.n.01"},{"index":7,"sense_key":"pound%1:23:04::","synset_id":"irish_pound.n.01"},{"index":8,"sense_key":"pound%1:23:03::","synset_id":"egyptian_pound.n.01"},{"index":9,"sense_key":"pound%1:23:02::","synset_id":"cypriot_pound.n.01"},{"index":10,"sense_key":"pound%1:23:00::","synset_id":"pound.n.10"},{"index":11,"sense_key":"pound%1:18:00::","synset_id":"pound.n.11"},{"index":12,"sense_key":"pound%1:10:00::","synset_id":"pound.n.12"},{"index":13,"sense_key":"pound%1:06:00::","synset_id":"pound.n.13"},{"index":14,"sense_key":"pound%1:04:00::","synset_id":"hammer.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pound%1:23:01::","call_ids":["semeval2013.d012.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pound%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":782,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003271,"input_uncached_usd":0.000039099999999999995,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.605390458018519}
{"item_id":"semeval2013.d012.s000.t002","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":312,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.000406,"input_uncached_usd":0.0000156,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.09154925000621}
{"item_id":"semeval2013.d012.s001.t000","gold_sense_keys":["trouble%1:09:00::","trouble%1:11:00::","trouble%1:12:03::"],"candidates":[{"index":1,"sense_key":"trouble%1:09:00::","synset_id":"trouble.n.01"},{"index":2,"sense_key":"trouble%1:11:01::","synset_id":"fuss.n.02"},{"index":3,"sense_key":"trouble%1:11:00::","synset_id":"trouble.n.03"},{"index":4,"sense_key":"trouble%1:04:02::","synset_id":"trouble.n.04"},{"index":5,"sense_key":"trouble%1:12:03::","synset_id":"worry.n.02"},{"index":6,"sense_key":"trouble%1:26:02::","synset_id":"trouble.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"trouble%1:09:00::","call_ids":["semeval2013.d012.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"trouble%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018465,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.436005584022496}
{"item_id":"semeval2013.d012.s001.t001","gold_sense_keys":["uproar%1:26:00::"],"candidates":[{"index":1,"sense_key":"uproar%1:26:00::","synset_id":"tumult.n.01"},{"index":2,"sense_key":"uproar%1:11:00::","synset_id":"hubbub.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"uproar%1:26:00::","call_ids":["semeval2013.d012.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"uproar%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038145,"input_uncached_usd":0.000016649999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.323275499977171}
{"item_id":"semeval2013.d012.s001.t003","gold_sense_keys":["pound%1:23:01::"],"candidates":[{"index":1,"sense_key":"pound%1:23:09::","synset_id":"pound.n.01"},{"index":2,"sense_key":"pound%1:23:01::","synset_id":"british_pound.n.01"},{"index":3,"sense_key":"pound%1:23:10::","synset_id":"pound.n.03"},{"index":4,"sense_key":"pound%1:23:08::","synset_id":"syrian_pound.n.01"},{"index":5,"sense_key":"pound%1:23:07::","synset_id":"sudanese_pound.n.01"},{"index":6,"sense_key":"pound%1:23:05::","synset_id":"lebanese_pound.n.01"},{"index":7,"sense_key":"pound%1:23:04::","synset_id":"irish_pound.n.01"},{"index":8,"sense_key":"pound%1:23:03::","synset_id":"egyptian_pound.n.01"},{"index":9,"sense_key":"pound%1:23:02::","synset_id":"cypriot_pound.n.01"},{"index":10,"sense_key":"pound%1:23:00::","synset_id":"pound.n.10"},{"index":11,"sense_key":"pound%1:18:00::","synset_id":"pound.n.11"},{"index":12,"sense_key":"pound%1:10:00::","synset_id":"pound.n.12"},{"index":13,"sense_key":"pound%1:06:00::","synset_id":"pound.n.13"},{"index":14,"sense_key":"pound%1:04:00::","synset_id":"hammer.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"pound%1:23:01::","call_ids":["semeval2013.d012.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"pound%1:23:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":844,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022779999999999998,"input_uncached_usd":0.000042199999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0258296659449115}
{"item_id":"semeval2013.d012.s001.t005","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":373,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017864999999999998,"input_uncached_usd":0.00001865,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7309714169823565}
{"item_id":"semeval2013.d012.s002.t000","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2013.d012.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020495,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.831295541021973}
{"item_id":"semeval2013.d012.s002.t001","gold_sense_keys":["home%1:15:02::"],"candidates":[{"index":1,"sense_key":"home%1:15:04::","synset_id":"home.n.01"},{"index":2,"sense_key":"home%1:06:00::","synset_id":"dwelling.n.01"},{"index":3,"sense_key":"home%1:15:02::","synset_id":"home.n.03"},{"index":4,"sense_key":"home%1:06:02::","synset_id":"home_plate.n.01"},{"index":5,"sense_key":"home%1:15:05::","synset_id":"base.n.14"},{"index":6,"sense_key":"home%1:15:03::","synset_id":"home.n.06"},{"index":7,"sense_key":"home%1:26:00::","synset_id":"home.n.07"},{"index":8,"sense_key":"home%1:14:00::","synset_id":"family.n.01"},{"index":9,"sense_key":"home%1:06:01::","synset_id":"home.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"home%1:15:02::","call_ids":["semeval2013.d012.s002.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"home%1:15:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":678,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002963,"input_uncached_usd":0.0000339,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.662440417043399}
{"item_id":"semeval2013.d012.s002.t002","gold_sense_keys":["industry%1:14:00::"],"candidates":[{"index":1,"sense_key":"industry%1:14:00::","synset_id":"industry.n.01"},{"index":2,"sense_key":"industry%1:04:00::","synset_id":"industry.n.02"},{"index":3,"sense_key":"industry%1:07:00::","synset_id":"diligence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industry%1:14:00::","call_ids":["semeval2013.d012.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001803,"input_uncached_usd":0.0000203,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2655878329533152}
{"item_id":"semeval2013.d012.s002.t003","gold_sense_keys":["giant%1:14:00::"],"candidates":[{"index":1,"sense_key":"giant%1:05:00::","synset_id":"giant.n.01"},{"index":2,"sense_key":"giant%1:18:01::","synset_id":"colossus.n.02"},{"index":3,"sense_key":"giant%1:14:00::","synset_id":"giant.n.03"},{"index":4,"sense_key":"giant%1:18:03::","synset_id":"giant.n.04"},{"index":5,"sense_key":"giant%1:18:00::","synset_id":"giant.n.05"},{"index":6,"sense_key":"giant%1:18:02::","synset_id":"giant.n.06"},{"index":7,"sense_key":"giant%1:17:00::","synset_id":"giant_star.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"giant%1:14:00::","call_ids":["semeval2013.d012.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"giant%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015984999999999998,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2679045000113547}
{"item_id":"semeval2013.d012.s002.t005","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":384,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001792,"input_uncached_usd":0.0000192,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.638325917010661}
{"item_id":"semeval2013.d012.s002.t006","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003345,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.498456667002756}
{"item_id":"semeval2013.d012.s002.t008","gold_sense_keys":["dispute%1:10:00::"],"candidates":[{"index":1,"sense_key":"dispute%1:10:00::","synset_id":"dispute.n.01"},{"index":2,"sense_key":"dispute%1:04:00::","synset_id":"dispute.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dispute%1:10:00::","call_ids":["semeval2013.d012.s002.t008__v1__a1","semeval2013.d012.s002.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dispute%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":2145,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00089505,"input_uncached_usd":0.00003704999999999999,"input_cached_usd":0.0,"output_usd":0.000858,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.862306124996394}
{"item_id":"semeval2013.d012.s002.t009","gold_sense_keys":["bankruptcy%1:26:00::"],"candidates":[{"index":1,"sense_key":"bankruptcy%1:26:01::","synset_id":"bankruptcy.n.01"},{"index":2,"sense_key":"bankruptcy%1:26:00::","synset_id":"bankruptcy.n.02"},{"index":3,"sense_key":"bankruptcy%1:04:00::","synset_id":"bankruptcy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bankruptcy%1:26:00::","call_ids":["semeval2013.d012.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bankruptcy%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00074255,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.429569832980633}
{"item_id":"semeval2013.d012.s003.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:00::","call_ids":["semeval2013.d012.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00015104999999999999,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.344973875035066}
{"item_id":"semeval2013.d012.s003.t001","gold_sense_keys":["sale%1:04:02::"],"candidates":[{"index":1,"sense_key":"sale%1:04:00::","synset_id":"sale.n.01"},{"index":2,"sense_key":"sale%1:04:02::","synset_id":"sale.n.02"},{"index":3,"sense_key":"sale%1:04:01::","synset_id":"sale.n.03"},{"index":4,"sense_key":"sale%1:26:00::","synset_id":"sale.n.04"},{"index":5,"sense_key":"sale%1:10:00::","synset_id":"sale.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sale%1:04:00::","call_ids":["semeval2013.d012.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sale%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":571,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029095,"input_uncached_usd":0.00002855,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.818267999973614}
{"item_id":"semeval2013.d012.s004.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"regulator%1:06:01::","call_ids":["semeval2013.d012.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"regulator%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002584,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.142242375004571}
{"item_id":"semeval2013.d012.s004.t001","gold_sense_keys":["string%1:14:00::"],"candidates":[{"index":1,"sense_key":"string%1:06:00::","synset_id":"string.n.01"},{"index":2,"sense_key":"string%1:06:05::","synset_id":"bowed_stringed_instrument.n.01"},{"index":3,"sense_key":"string%1:06:03::","synset_id":"string.n.03"},{"index":4,"sense_key":"string%1:14:00::","synset_id":"string.n.04"},{"index":5,"sense_key":"string%1:10:00::","synset_id":"string.n.05"},{"index":6,"sense_key":"string%1:06:01::","synset_id":"drawstring.n.01"},{"index":7,"sense_key":"string%1:27:00::","synset_id":"string.n.07"},{"index":8,"sense_key":"string%1:17:00::","synset_id":"string.n.08"},{"index":9,"sense_key":"string%1:06:04::","synset_id":"string.n.09"},{"index":10,"sense_key":"string%1:06:02::","synset_id":"chain.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"string%1:14:00::","call_ids":["semeval2013.d012.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"string%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":708,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002466,"input_uncached_usd":0.0000354,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.351912332989741}
{"item_id":"semeval2013.d012.s004.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0004034,"input_uncached_usd":0.0000382,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.472539416979998}
{"item_id":"semeval2013.d012.s004.t003","gold_sense_keys":["promise%1:10:00::"],"candidates":[{"index":1,"sense_key":"promise%1:10:00::","synset_id":"promise.n.01"},{"index":2,"sense_key":"promise%1:09:00::","synset_id":"promise.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"promise%1:10:00::","call_ids":["semeval2013.d012.s004.t003__v1__a1","semeval2013.d012.s004.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"promise%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043904999999999997,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.914214958029334}
{"item_id":"semeval2013.d012.s005.t000","gold_sense_keys":["value%1:21:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"value%1:21:00::","call_ids":["semeval2013.d012.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"value%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016429999999999998,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.904279541980941}
{"item_id":"semeval2013.d012.s005.t001","gold_sense_keys":["security%1:21:04::"],"candidates":[{"index":1,"sense_key":"security%1:26:00::","synset_id":"security.n.01"},{"index":2,"sense_key":"security%1:21:01::","synset_id":"security.n.02"},{"index":3,"sense_key":"security%1:12:00::","synset_id":"security.n.03"},{"index":4,"sense_key":"security%1:21:04::","synset_id":"security.n.04"},{"index":5,"sense_key":"security%1:21:00::","synset_id":"security.n.05"},{"index":6,"sense_key":"security%1:14:00::","synset_id":"security.n.06"},{"index":7,"sense_key":"security%1:10:00::","synset_id":"security.n.07"},{"index":8,"sense_key":"security%1:06:00::","synset_id":"security_system.n.02"},{"index":9,"sense_key":"security%1:04:00::","synset_id":"security.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"security%1:21:04::","call_ids":["semeval2013.d012.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"security%1:21:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032485,"input_uncached_usd":0.00003685,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.643054457963444}
{"item_id":"semeval2013.d012.s005.t002","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d012.s005.t002__v1__a1","semeval2013.d012.s005.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":893,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005182499999999999,"input_uncached_usd":0.00004465,"input_cached_usd":0.0,"output_usd":0.0004736,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.806055624037981}
{"item_id":"semeval2013.d012.s006.t000","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"issue%1:09:01::","call_ids":["semeval2013.d012.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"issue%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":888,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005376,"input_uncached_usd":0.000044399999999999995,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.653445250005461}
{"item_id":"semeval2013.d012.s006.t004","gold_sense_keys":["new_york%1:15:00::","new_york%1:15:01::"],"candidates":[{"index":1,"sense_key":"new_york%1:15:01::","synset_id":"new_york.n.01"},{"index":2,"sense_key":"new_york%1:15:00::","synset_id":"new_york.n.02"},{"index":3,"sense_key":"new_york%1:15:02::","synset_id":"new_york.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new_york%1:15:01::","call_ids":["semeval2013.d012.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new_york%1:15:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005422000000000001,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.278007499990053}
{"item_id":"semeval2013.d012.s007.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":774,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003267,"input_uncached_usd":0.0000387,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.75959245901322}
{"item_id":"semeval2013.d012.s007.t002","gold_sense_keys":["settlement%1:10:00::"],"candidates":[{"index":1,"sense_key":"settlement%1:14:00::","synset_id":"colony.n.01"},{"index":2,"sense_key":"settlement%1:14:01::","synset_id":"village.n.01"},{"index":3,"sense_key":"settlement%1:10:00::","synset_id":"settlement.n.03"},{"index":4,"sense_key":"settlement%1:04:01::","synset_id":"colonization.n.01"},{"index":5,"sense_key":"settlement%1:09:00::","synset_id":"settlement.n.05"},{"index":6,"sense_key":"settlement%1:15:00::","synset_id":"settlement.n.06"},{"index":7,"sense_key":"settlement%1:04:00::","synset_id":"liquidation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"settlement%1:09:00::","call_ids":["semeval2013.d012.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"settlement%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029335,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.749023374984972}
{"item_id":"semeval2013.d012.s008.t000","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":438,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001567,"input_uncached_usd":0.0000219,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0959619589848444}
{"item_id":"semeval2013.d012.s009.t001","gold_sense_keys":["responsiveness%1:07:00::"],"candidates":[{"index":1,"sense_key":"responsiveness%1:09:00::","synset_id":"responsiveness.n.01"},{"index":2,"sense_key":"responsiveness%1:07:00::","synset_id":"responsiveness.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"responsiveness%1:07:00::","call_ids":["semeval2013.d012.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"responsiveness%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.000665,"input_uncached_usd":0.000018599999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.72955779201584}
{"item_id":"semeval2013.d012.s009.t002","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":746,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014609999999999997,"input_uncached_usd":0.0000373,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1097463330370374}
{"item_id":"semeval2013.d012.s010.t002","gold_sense_keys":["bankruptcy%1:04:00::"],"candidates":[{"index":1,"sense_key":"bankruptcy%1:26:01::","synset_id":"bankruptcy.n.01"},{"index":2,"sense_key":"bankruptcy%1:26:00::","synset_id":"bankruptcy.n.02"},{"index":3,"sense_key":"bankruptcy%1:04:00::","synset_id":"bankruptcy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bankruptcy%1:04:00::","call_ids":["semeval2013.d012.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bankruptcy%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010345,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1100902920006774}
{"item_id":"semeval2013.d012.s010.t004","gold_sense_keys":["deposit%1:21:01::"],"candidates":[{"index":1,"sense_key":"deposit%1:19:00::","synset_id":"deposit.n.01"},{"index":2,"sense_key":"deposit%1:17:00::","synset_id":"sediment.n.01"},{"index":3,"sense_key":"deposit%1:22:00::","synset_id":"deposition.n.01"},{"index":4,"sense_key":"deposit%1:21:01::","synset_id":"deposit.n.04"},{"index":5,"sense_key":"deposit%1:21:00::","synset_id":"down_payment.n.01"},{"index":6,"sense_key":"deposit%1:21:02::","synset_id":"deposit.n.06"},{"index":7,"sense_key":"deposit%1:10:01::","synset_id":"deposit.n.07"},{"index":8,"sense_key":"deposit%1:06:00::","synset_id":"depository.n.01"},{"index":9,"sense_key":"deposit%1:04:00::","synset_id":"deposit.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"deposit%1:22:00::","call_ids":["semeval2013.d012.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"deposit%1:22:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002915,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.369703333999496}
{"item_id":"semeval2013.d012.s011.t000","gold_sense_keys":["compensation%1:21:00::"],"candidates":[{"index":1,"sense_key":"compensation%1:21:00::","synset_id":"compensation.n.01"},{"index":2,"sense_key":"compensation%1:22:00::","synset_id":"compensation.n.02"},{"index":3,"sense_key":"compensation%1:04:00::","synset_id":"recompense.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compensation%1:21:00::","call_ids":["semeval2013.d012.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compensation%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001281,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5178429169463925}
{"item_id":"semeval2013.d012.s011.t002","gold_sense_keys":["billion%1:23:00::"],"candidates":[{"index":1,"sense_key":"billion%1:23:01::","synset_id":"billion.n.01"},{"index":2,"sense_key":"billion%1:23:02::","synset_id":"million.n.02"},{"index":3,"sense_key":"billion%1:23:00::","synset_id":"billion.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"billion%1:23:02::","call_ids":["semeval2013.d012.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"billion%1:23:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005395,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.106297583959531}
{"item_id":"semeval2013.d012.s012.t000","gold_sense_keys":["settlement%1:10:00::"],"candidates":[{"index":1,"sense_key":"settlement%1:14:00::","synset_id":"colony.n.01"},{"index":2,"sense_key":"settlement%1:14:01::","synset_id":"village.n.01"},{"index":3,"sense_key":"settlement%1:10:00::","synset_id":"settlement.n.03"},{"index":4,"sense_key":"settlement%1:04:01::","synset_id":"colonization.n.01"},{"index":5,"sense_key":"settlement%1:09:00::","synset_id":"settlement.n.05"},{"index":6,"sense_key":"settlement%1:15:00::","synset_id":"settlement.n.06"},{"index":7,"sense_key":"settlement%1:04:00::","synset_id":"liquidation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"settlement%1:10:00::","call_ids":["semeval2013.d012.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"settlement%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052105,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.499044208030682}
{"item_id":"semeval2013.d012.s013.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006658,"input_uncached_usd":0.0000194,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.738501625019126}
{"item_id":"semeval2013.d012.s013.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"bank%1:17:00::","call_ids":["semeval2013.d012.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"bank%1:17:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055435,"input_uncached_usd":0.00003595,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.201186250022147}
{"item_id":"semeval2013.d012.s013.t002","gold_sense_keys":["summer%1:28:00::"],"candidates":[{"index":1,"sense_key":"summer%1:28:00::","synset_id":"summer.n.01"},{"index":2,"sense_key":"summer%1:28:01::","synset_id":"summer.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"summer%1:28:00::","call_ids":["semeval2013.d012.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"summer%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":368,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002296,"input_uncached_usd":0.0000184,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.076711417001206}
{"item_id":"semeval2013.d012.s016.t000","gold_sense_keys":["fate%1:11:00::"],"candidates":[{"index":1,"sense_key":"fate%1:11:00::","synset_id":"destiny.n.01"},{"index":2,"sense_key":"fate%1:18:00::","synset_id":"destiny.n.02"},{"index":3,"sense_key":"fate%1:26:00::","synset_id":"fortune.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"fate%1:26:00::","call_ids":["semeval2013.d012.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"fate%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":417,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00061605,"input_uncached_usd":0.00002085,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.247019166999962}
{"item_id":"semeval2013.d012.s016.t001","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d012.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024915,"input_uncached_usd":0.00003755,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.583039958029985}
{"item_id":"semeval2013.d012.s016.t002","gold_sense_keys":["company%1:14:01::"],"candidates":[{"index":1,"sense_key":"company%1:14:01::","synset_id":"company.n.01"},{"index":2,"sense_key":"company%1:14:03::","synset_id":"company.n.02"},{"index":3,"sense_key":"company%1:26:00::","synset_id":"company.n.03"},{"index":4,"sense_key":"company%1:14:02::","synset_id":"company.n.04"},{"index":5,"sense_key":"company%1:18:00::","synset_id":"caller.n.01"},{"index":6,"sense_key":"company%1:14:00::","synset_id":"company.n.06"},{"index":7,"sense_key":"company%1:14:06::","synset_id":"party.n.03"},{"index":8,"sense_key":"company%1:14:04::","synset_id":"ship's_company.n.01"},{"index":9,"sense_key":"company%1:14:05::","synset_id":"company.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"company%1:14:01::","call_ids":["semeval2013.d012.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"company%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016529999999999998,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.604865000001155}
{"item_id":"semeval2013.d012.s017.t002","gold_sense_keys":["height%1:26:00::"],"candidates":[{"index":1,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":2,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":3,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":4,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"height%1:26:00::","call_ids":["semeval2013.d012.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"height%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":273,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001314,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.0001092,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.282566833018791}
{"item_id":"semeval2013.d012.s017.t003","gold_sense_keys":["crisis%1:26:00::"],"candidates":[{"index":1,"sense_key":"crisis%1:26:00::","synset_id":"crisis.n.01"},{"index":2,"sense_key":"crisis%1:11:00::","synset_id":"crisis.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"crisis%1:26:00::","call_ids":["semeval2013.d012.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"crisis%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":341,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012585,"input_uncached_usd":0.000017049999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.048044541035779}
{"item_id":"semeval2013.d012.s018.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034935,"input_uncached_usd":0.00003575,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.623900874983519}
{"item_id":"semeval2013.d012.s018.t001","gold_sense_keys":["wind%1:19:01::"],"candidates":[{"index":1,"sense_key":"wind%1:19:00::","synset_id":"wind.n.01"},{"index":2,"sense_key":"wind%1:19:01::","synset_id":"wind.n.02"},{"index":3,"sense_key":"wind%1:04:01::","synset_id":"wind.n.03"},{"index":4,"sense_key":"wind%1:10:01::","synset_id":"wind.n.04"},{"index":5,"sense_key":"wind%1:10:00::","synset_id":"tip.n.03"},{"index":6,"sense_key":"wind%1:06:00::","synset_id":"wind_instrument.n.01"},{"index":7,"sense_key":"wind%1:04:02::","synset_id":"fart.n.01"},{"index":8,"sense_key":"wind%1:04:00::","synset_id":"wind.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wind%1:19:01::","call_ids":["semeval2013.d012.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wind%1:19:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002941,"input_uncached_usd":0.0000317,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.01577537500998}
{"item_id":"semeval2013.d012.s018.t002","gold_sense_keys":["usa%1:15:00::"],"candidates":[{"index":1,"sense_key":"usa%1:15:00::","synset_id":"united_states.n.01"},{"index":2,"sense_key":"usa%1:14:00::","synset_id":"united_states_army.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"usa%1:15:00::","call_ids":["semeval2013.d012.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"usa%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017949999999999997,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3157668340136297}
{"item_id":"semeval2013.d012.s018.t003","gold_sense_keys":["year%1:28:01::"],"candidates":[{"index":1,"sense_key":"year%1:28:01::","synset_id":"year.n.01"},{"index":2,"sense_key":"year%1:28:02::","synset_id":"year.n.02"},{"index":3,"sense_key":"year%1:28:00::","synset_id":"year.n.03"},{"index":4,"sense_key":"year%1:14:00::","synset_id":"class.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"year%1:28:01::","call_ids":["semeval2013.d012.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"year%1:28:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018334999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7043635419686325}
{"item_id":"semeval2013.d012.s019.t001","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012759999999999998,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.076359041966498}
{"item_id":"semeval2013.d012.s019.t002","gold_sense_keys":["number%1:07:00::"],"candidates":[{"index":1,"sense_key":"number%1:07:00::","synset_id":"number.n.01"},{"index":2,"sense_key":"number%1:23:00::","synset_id":"number.n.02"},{"index":3,"sense_key":"number%1:10:04::","synset_id":"act.n.04"},{"index":4,"sense_key":"number%1:10:03::","synset_id":"phone_number.n.01"},{"index":5,"sense_key":"number%1:10:01::","synset_id":"numeral.n.01"},{"index":6,"sense_key":"number%1:10:00::","synset_id":"issue.n.02"},{"index":7,"sense_key":"number%1:14:00::","synset_id":"number.n.07"},{"index":8,"sense_key":"number%1:10:02::","synset_id":"number.n.08"},{"index":9,"sense_key":"number%1:07:01::","synset_id":"number.n.09"},{"index":10,"sense_key":"number%1:10:05::","synset_id":"number.n.10"},{"index":11,"sense_key":"number%1:06:00::","synset_id":"number.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"number%1:07:00::","call_ids":["semeval2013.d012.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"number%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":748,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002742,"input_uncached_usd":0.0000374,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.650925250025466}
{"item_id":"semeval2013.d012.s019.t003","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"claim%1:04:00::","call_ids":["semeval2013.d012.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"claim%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015939999999999997,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5322551249992102}
{"item_id":"semeval2013.d012.s019.t004","gold_sense_keys":["compensation%1:21:00::"],"candidates":[{"index":1,"sense_key":"compensation%1:21:00::","synset_id":"compensation.n.01"},{"index":2,"sense_key":"compensation%1:22:00::","synset_id":"compensation.n.02"},{"index":3,"sense_key":"compensation%1:04:00::","synset_id":"recompense.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compensation%1:21:00::","call_ids":["semeval2013.d012.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compensation%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":376,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002304,"input_uncached_usd":0.0000188,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.070478874957189}
{"item_id":"semeval2013.d012.s019.t005","gold_sense_keys":["misconduct%1:04:00::"],"candidates":[{"index":1,"sense_key":"misconduct%1:04:01::","synset_id":"misconduct.n.01"},{"index":2,"sense_key":"misconduct%1:04:00::","synset_id":"wrongdoing.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"misconduct%1:04:00::","call_ids":["semeval2013.d012.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"misconduct%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":341,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035624999999999996,"input_uncached_usd":0.000017049999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.293844084022567}
{"item_id":"semeval2013.d012.s020.t000","gold_sense_keys":["wave%1:04:02::"],"candidates":[{"index":1,"sense_key":"wave%1:11:01::","synset_id":"wave.n.01"},{"index":2,"sense_key":"wave%1:04:02::","synset_id":"wave.n.02"},{"index":3,"sense_key":"wave%1:11:00::","synset_id":"wave.n.03"},{"index":4,"sense_key":"wave%1:11:02::","synset_id":"wave.n.04"},{"index":5,"sense_key":"wave%1:10:01::","synset_id":"wave.n.05"},{"index":6,"sense_key":"wave%1:08:00::","synset_id":"wave.n.06"},{"index":7,"sense_key":"wave%1:25:00::","synset_id":"wave.n.07"},{"index":8,"sense_key":"wave%1:19:00::","synset_id":"wave.n.08"},{"index":9,"sense_key":"wave%1:18:00::","synset_id":"wave.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"wave%1:04:02::","call_ids":["semeval2013.d012.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"wave%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024155,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.942910958023276}
{"item_id":"semeval2013.d012.s020.t001","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"claim%1:04:00::","call_ids":["semeval2013.d012.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"claim%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021155,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.110586417024024}
{"item_id":"semeval2013.d012.s021.t000","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037674999999999997,"input_uncached_usd":0.00003755,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.26223791600205}
{"item_id":"semeval2013.d012.s021.t007","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s021.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":459,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015734999999999998,"input_uncached_usd":0.00002295,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.875632332987152}
{"item_id":"semeval2013.d012.s022.t002","gold_sense_keys":["value%1:21:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"value%1:21:00::","call_ids":["semeval2013.d012.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"value%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011144999999999998,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9504262499976903}
{"item_id":"semeval2013.d012.s022.t003","gold_sense_keys":["dollar%1:23:00::"],"candidates":[{"index":1,"sense_key":"dollar%1:23:00::","synset_id":"dollar.n.01"},{"index":2,"sense_key":"dollar%1:21:00::","synset_id":"dollar.n.02"},{"index":3,"sense_key":"dollar%1:21:01::","synset_id":"dollar.n.03"},{"index":4,"sense_key":"dollar%1:10:00::","synset_id":"dollar.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dollar%1:23:00::","call_ids":["semeval2013.d012.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dollar%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023444999999999998,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8423130420269445}
{"item_id":"semeval2013.d012.s023.t000","gold_sense_keys":["regulator%1:18:00::"],"candidates":[{"index":1,"sense_key":"regulator%1:06:00::","synset_id":"regulator.n.01"},{"index":2,"sense_key":"regulator%1:18:00::","synset_id":"regulator.n.02"},{"index":3,"sense_key":"regulator%1:06:01::","synset_id":"governor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regulator%1:18:00::","call_ids":["semeval2013.d012.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regulator%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002834,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.714078499993775}
{"item_id":"semeval2013.d012.s023.t001","gold_sense_keys":["bank%1:14:00::"],"candidates":[{"index":1,"sense_key":"bank%1:17:01::","synset_id":"bank.n.01"},{"index":2,"sense_key":"bank%1:14:00::","synset_id":"depository_financial_institution.n.01"},{"index":3,"sense_key":"bank%1:17:00::","synset_id":"bank.n.03"},{"index":4,"sense_key":"bank%1:14:01::","synset_id":"bank.n.04"},{"index":5,"sense_key":"bank%1:21:00::","synset_id":"bank.n.05"},{"index":6,"sense_key":"bank%1:21:01::","synset_id":"bank.n.06"},{"index":7,"sense_key":"bank%1:17:02::","synset_id":"bank.n.07"},{"index":8,"sense_key":"bank%1:06:01::","synset_id":"savings_bank.n.02"},{"index":9,"sense_key":"bank%1:06:00::","synset_id":"bank.n.09"},{"index":10,"sense_key":"bank%1:04:00::","synset_id":"bank.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"bank%1:14:00::","call_ids":["semeval2013.d012.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"bank%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":751,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017194999999999998,"input_uncached_usd":0.00003755,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3881873339996673}
{"item_id":"semeval2013.d012.s023.t002","gold_sense_keys":["responsibility%1:07:00::"],"candidates":[{"index":1,"sense_key":"responsibility%1:04:00::","synset_id":"duty.n.01"},{"index":2,"sense_key":"responsibility%1:26:00::","synset_id":"province.n.02"},{"index":3,"sense_key":"responsibility%1:07:00::","synset_id":"responsibility.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"responsibility%1:07:00::","call_ids":["semeval2013.d012.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"responsibility%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":485,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020984999999999998,"input_uncached_usd":0.00002425,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5354997499962337}
{"item_id":"semeval2013.d012.s023.t004","gold_sense_keys":["sum%1:21:00::"],"candidates":[{"index":1,"sense_key":"sum%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"sum%1:09:01::","synset_id":"sum.n.02"},{"index":3,"sense_key":"sum%1:14:00::","synset_id":"sum.n.03"},{"index":4,"sense_key":"sum%1:09:00::","synset_id":"kernel.n.03"},{"index":5,"sense_key":"sum%1:06:00::","synset_id":"sum.n.05"},{"index":6,"sense_key":"sum%1:14:01::","synset_id":"union.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"sum%1:21:00::","call_ids":["semeval2013.d012.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"sum%1:21:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002653,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.12853216700023}
{"item_id":"semeval2013.d012.s024.t000","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"claim%1:04:00::","call_ids":["semeval2013.d012.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"claim%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034015,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.372571958985645}
{"item_id":"semeval2013.d012.s024.t001","gold_sense_keys":["intention%1:09:00::"],"candidates":[{"index":1,"sense_key":"intention%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"intention%1:09:01::","synset_id":"intention.n.02"},{"index":3,"sense_key":"intention%1:04:00::","synset_id":"intention.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"intention%1:09:00::","call_ids":["semeval2013.d012.s024.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"intention%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038655,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.318073707981966}
{"item_id":"semeval2013.d012.s024.t002","gold_sense_keys":["claim%1:04:00::","claim%1:10:00::"],"candidates":[{"index":1,"sense_key":"claim%1:10:00::","synset_id":"claim.n.01"},{"index":2,"sense_key":"claim%1:10:02::","synset_id":"claim.n.02"},{"index":3,"sense_key":"claim%1:04:00::","synset_id":"claim.n.03"},{"index":4,"sense_key":"claim%1:07:02::","synset_id":"claim.n.04"},{"index":5,"sense_key":"claim%1:07:01::","synset_id":"title.n.07"},{"index":6,"sense_key":"claim%1:10:01::","synset_id":"call.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"claim%1:10:00::","call_ids":["semeval2013.d012.s024.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"claim%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":531,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00095455,"input_uncached_usd":0.000026549999999999998,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.66849045897834}
{"item_id":"semeval2015.d000.s000.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2015.d000.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":322,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025289999999999997,"input_uncached_usd":0.0000161,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.779214832989965}
{"item_id":"semeval2015.d000.s000.t003","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d000.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":271,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048074999999999995,"input_uncached_usd":0.000013549999999999999,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.845321083033923}
{"item_id":"semeval2015.d000.s000.t004","gold_sense_keys":["assessment%1:04:00::"],"candidates":[{"index":1,"sense_key":"assessment%1:09:00::","synset_id":"appraisal.n.01"},{"index":2,"sense_key":"assessment%1:21:00::","synset_id":"assessment.n.02"},{"index":3,"sense_key":"assessment%1:07:00::","synset_id":"assessment.n.03"},{"index":4,"sense_key":"assessment%1:04:00::","synset_id":"judgment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"assessment%1:04:00::","call_ids":["semeval2015.d000.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"assessment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003302,"input_uncached_usd":0.0000166,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.214481999981217}
{"item_id":"semeval2015.d000.s000.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d000.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000314,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.113832041039132}
{"item_id":"semeval2015.d000.s001.t000","gold_sense_keys":["explain%2:32:00::","explain%2:32:01::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"explain%2:32:00::","call_ids":["semeval2015.d000.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"explain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012695,"input_uncached_usd":0.00001815,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.344604999991134}
{"item_id":"semeval2015.d000.s001.t001","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d000.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020089999999999998,"input_uncached_usd":0.0000153,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.565558209025767}
{"item_id":"semeval2015.d000.s001.t003","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"product%1:06:01::","call_ids":["semeval2015.d000.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"product%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.000339,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.250188999983948}
{"item_id":"semeval2015.d000.s001.t004","gold_sense_keys":["human%3:01:01::"],"candidates":[{"index":1,"sense_key":"human%3:01:00::","synset_id":"human.a.01"},{"index":2,"sense_key":"human%3:01:01::","synset_id":"human.a.02"},{"index":3,"sense_key":"human%3:00:00::","synset_id":"human.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"human%3:01:01::","call_ids":["semeval2015.d000.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"human%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":332,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003558,"input_uncached_usd":0.0000166,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.127870208001696}
{"item_id":"semeval2015.d000.s001.t005","gold_sense_keys":["use%1:04:00::","use%1:07:02::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034284999999999996,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.123196000000462}
{"item_id":"semeval2015.d000.s001.t006","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d000.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":408,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000206,"input_uncached_usd":0.000020399999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.828586291987449}
{"item_id":"semeval2015.d000.s001.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":643,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039694999999999997,"input_uncached_usd":0.000032149999999999995,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.325780999963172}
{"item_id":"semeval2015.d000.s001.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perform%2:36:00::","call_ids":["semeval2015.d000.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":388,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001282,"input_uncached_usd":0.0000194,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.401146041986067}
{"item_id":"semeval2015.d000.s001.t009","gold_sense_keys":["reach%2:38:06::","reach%2:41:00::"],"candidates":[{"index":1,"sense_key":"reach%2:38:01::","synset_id":"reach.v.01"},{"index":2,"sense_key":"reach%2:38:00::","synset_id":"reach.v.02"},{"index":3,"sense_key":"reach%2:35:00::","synset_id":"reach.v.03"},{"index":4,"sense_key":"reach%2:32:00::","synset_id":"reach.v.04"},{"index":5,"sense_key":"reach%2:41:00::","synset_id":"achieve.v.01"},{"index":6,"sense_key":"reach%2:42:00::","synset_id":"reach.v.06"},{"index":7,"sense_key":"reach%2:38:06::","synset_id":"reach.v.07"},{"index":8,"sense_key":"reach%2:40:00::","synset_id":"pass.v.05"},{"index":9,"sense_key":"reach%2:33:00::","synset_id":"strive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reach%2:38:01::","call_ids":["semeval2015.d000.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reach%2:38:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":2512,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.0010374499999999999,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0010048,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.412787957990076}
{"item_id":"semeval2015.d000.s001.t010","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"},{"index":3,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d000.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007575,"input_uncached_usd":0.00001815,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.5674486249918118}
{"item_id":"semeval2015.d000.s001.t011","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007487,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.748031666968018}
{"item_id":"semeval2015.d000.s001.t012","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015439999999999999,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.792382000014186}
{"item_id":"semeval2015.d000.s002.t000","gold_sense_keys":["need%2:34:00::","need%2:34:01::"],"candidates":[{"index":1,"sense_key":"need%2:42:00::","synset_id":"necessitate.v.01"},{"index":2,"sense_key":"need%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"need%2:34:01::","synset_id":"need.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"need%2:34:01::","call_ids":["semeval2015.d000.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"need%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004094,"input_uncached_usd":0.000018999999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.137451666989364}
{"item_id":"semeval2015.d000.s002.t001","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s002.t001__v1__a1","semeval2015.d000.s002.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":757,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003834499999999999,"input_uncached_usd":0.00003785,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5205240830546245}
{"item_id":"semeval2015.d000.s002.t002","gold_sense_keys":["information%1:09:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":2,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":3,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":4,"sense_key":"information%1:14:00::","synset_id":"data.n.01"},{"index":5,"sense_key":"information%1:07:00::","synset_id":"information.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"information%1:09:00::","call_ids":["semeval2015.d000.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"information%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030845,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.5857299999915995}
{"item_id":"semeval2015.d000.s002.t003","gold_sense_keys":["medical%3:00:00::"],"candidates":[{"index":1,"sense_key":"medical%3:01:00::","synset_id":"medical.a.01"},{"index":2,"sense_key":"medical%3:00:00::","synset_id":"medical.a.02"},{"index":3,"sense_key":"medical%3:01:02::","synset_id":"aesculapian.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medical%3:00:00::","call_ids":["semeval2015.d000.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medical%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040885,"input_uncached_usd":0.000018449999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.103472999995574}
{"item_id":"semeval2015.d000.s002.t004","gold_sense_keys":["condition%1:26:05::"],"candidates":[{"index":1,"sense_key":"condition%1:26:00::","synset_id":"condition.n.01"},{"index":2,"sense_key":"condition%1:10:01::","synset_id":"condition.n.02"},{"index":3,"sense_key":"condition%1:26:01::","synset_id":"condition.n.03"},{"index":4,"sense_key":"condition%1:09:00::","synset_id":"circumstance.n.03"},{"index":5,"sense_key":"condition%1:26:02::","synset_id":"condition.n.05"},{"index":6,"sense_key":"condition%1:26:05::","synset_id":"condition.n.06"},{"index":7,"sense_key":"condition%1:10:02::","synset_id":"condition.n.07"},{"index":8,"sense_key":"condition%1:09:01::","synset_id":"condition.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"condition%1:26:05::","call_ids":["semeval2015.d000.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"condition%1:26:05::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":602,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002669,"input_uncached_usd":0.0000301,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.868718667014036}
{"item_id":"semeval2015.d000.s002.t005","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002071,"input_uncached_usd":0.000021499999999999997,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7154785419697873}
{"item_id":"semeval2015.d000.s002.t006","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["semeval2015.d000.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005295,"input_uncached_usd":0.0000367,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.438549541984685}
{"item_id":"semeval2015.d000.s002.t008","gold_sense_keys":["leaflet%1:10:00::"],"candidates":[{"index":1,"sense_key":"leaflet%1:08:00::","synset_id":"cusp.n.02"},{"index":2,"sense_key":"leaflet%1:20:00::","synset_id":"leaflet.n.02"},{"index":3,"sense_key":"leaflet%1:10:00::","synset_id":"booklet.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"leaflet%1:10:00::","call_ids":["semeval2015.d000.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"leaflet%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012599999999999997,"input_uncached_usd":0.000017199999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.755654917040374}
{"item_id":"semeval2015.d000.s002.t010","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2015.d000.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":847,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025354999999999996,"input_uncached_usd":0.00004235,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.147707291995175}
{"item_id":"semeval2015.d000.s002.t011","gold_sense_keys":["contact%2:32:00::"],"candidates":[{"index":1,"sense_key":"contact%2:32:00::","synset_id":"reach.v.04"},{"index":2,"sense_key":"contact%2:35:03::","synset_id":"touch.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contact%2:32:00::","call_ids":["semeval2015.d000.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contact%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":353,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035685,"input_uncached_usd":0.00001765,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.581572583003435}
{"item_id":"semeval2015.d000.s002.t012","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":3,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"doctor%1:04:00::","call_ids":["semeval2015.d000.s002.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"doctor%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026005,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.155575417040382}
{"item_id":"semeval2015.d000.s003.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d000.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046444999999999993,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.056361125025433}
{"item_id":"semeval2015.d000.s003.t001","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00040845,"input_uncached_usd":0.000018049999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.091909042035695}
{"item_id":"semeval2015.d000.s003.t002","gold_sense_keys":["information%1:09:00::","information%1:14:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":2,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":3,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":4,"sense_key":"information%1:14:00::","synset_id":"data.n.01"},{"index":5,"sense_key":"information%1:07:00::","synset_id":"information.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"information%1:09:00::","call_ids":["semeval2015.d000.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"information%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043674999999999996,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.306344625016209}
{"item_id":"semeval2015.d000.s003.t003","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2015.d000.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028254999999999996,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.1010300000198185}
{"item_id":"semeval2015.d000.s003.t004","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"},{"index":3,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d000.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033329999999999997,"input_uncached_usd":0.000019699999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1524300840101205}
{"item_id":"semeval2015.d000.s003.t005","gold_sense_keys":["read%2:31:00::"],"candidates":[{"index":1,"sense_key":"read%2:31:00::","synset_id":"read.v.01"},{"index":2,"sense_key":"read%2:42:00::","synset_id":"read.v.02"},{"index":3,"sense_key":"read%2:31:01::","synset_id":"read.v.03"},{"index":4,"sense_key":"read%2:31:06::","synset_id":"read.v.04"},{"index":5,"sense_key":"read%2:31:02::","synset_id":"read.v.05"},{"index":6,"sense_key":"read%2:31:09::","synset_id":"take.v.06"},{"index":7,"sense_key":"read%2:31:05::","synset_id":"learn.v.04"},{"index":8,"sense_key":"read%2:32:02::","synset_id":"read.v.08"},{"index":9,"sense_key":"read%2:36:05::","synset_id":"read.v.09"},{"index":10,"sense_key":"read%2:31:08::","synset_id":"read.v.10"},{"index":11,"sense_key":"read%2:31:04::","synset_id":"understand.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"read%2:31:00::","call_ids":["semeval2015.d000.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"read%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040179999999999995,"input_uncached_usd":0.000037,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.162454625009559}
{"item_id":"semeval2015.d000.s003.t007","gold_sense_keys":["discussion%1:10:02::"],"candidates":[{"index":1,"sense_key":"discussion%1:10:02::","synset_id":"discussion.n.01"},{"index":2,"sense_key":"discussion%1:10:00::","synset_id":"discussion.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discussion%1:10:02::","call_ids":["semeval2015.d000.s003.t007__v1__a1","semeval2015.d000.s003.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discussion%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":781,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058945,"input_uncached_usd":0.00003905,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.944337583961897}
{"item_id":"semeval2015.d000.s003.t009","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2015.d000.s003.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025384999999999996,"input_uncached_usd":0.00004265,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7351502500241622}
{"item_id":"semeval2015.d000.s005.t000","gold_sense_keys":["powder%1:06:01::","powder%1:27:00::"],"candidates":[{"index":1,"sense_key":"powder%1:27:00::","synset_id":"powder.n.01"},{"index":2,"sense_key":"powder%1:27:01::","synset_id":"gunpowder.n.01"},{"index":3,"sense_key":"powder%1:06:01::","synset_id":"powder.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"powder%1:06:01::","call_ids":["semeval2015.d000.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"powder%1:06:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010455,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9594455840415321}
{"item_id":"semeval2015.d000.s005.t002","gold_sense_keys":["solution%1:27:00::"],"candidates":[{"index":1,"sense_key":"solution%1:27:00::","synset_id":"solution.n.01"},{"index":2,"sense_key":"solution%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"solution%1:09:00::","synset_id":"solution.n.03"},{"index":4,"sense_key":"solution%1:14:00::","synset_id":"solution.n.04"},{"index":5,"sense_key":"solution%1:04:00::","synset_id":"solution.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"solution%1:09:00::","call_ids":["semeval2015.d000.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"solution%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":498,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036449999999999997,"input_uncached_usd":0.0000249,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.422057874966413}
{"item_id":"semeval2015.d000.s005.t003","gold_sense_keys":["infusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"infusion%1:27:00::","synset_id":"infusion.n.01"},{"index":2,"sense_key":"infusion%1:22:00::","synset_id":"infusion.n.02"},{"index":3,"sense_key":"infusion%1:04:00::","synset_id":"infusion.n.03"},{"index":4,"sense_key":"infusion%1:04:01::","synset_id":"infusion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"infusion%1:04:00::","call_ids":["semeval2015.d000.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"infusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018309999999999998,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.957232207991183}
{"item_id":"semeval2015.d000.s005.t004","gold_sense_keys":["drip%2:35:10::"],"candidates":[{"index":1,"sense_key":"drip%2:38:00::","synset_id":"drip.v.01"},{"index":2,"sense_key":"drip%2:35:10::","synset_id":"dribble.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"drip%2:35:10::","call_ids":["semeval2015.d000.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"drip%2:35:10::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":361,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006132500000000001,"input_uncached_usd":0.000018049999999999998,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.586311499995645}
{"item_id":"semeval2015.d000.s005.t005","gold_sense_keys":["vein%1:08:00::"],"candidates":[{"index":1,"sense_key":"vein%1:08:00::","synset_id":"vein.n.01"},{"index":2,"sense_key":"vein%1:10:00::","synset_id":"vein.n.02"},{"index":3,"sense_key":"vein%1:20:00::","synset_id":"vein.n.03"},{"index":4,"sense_key":"vein%1:17:00::","synset_id":"vein.n.04"},{"index":5,"sense_key":"vein%1:05:00::","synset_id":"vein.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"vein%1:08:00::","call_ids":["semeval2015.d000.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"vein%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":493,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018465,"input_uncached_usd":0.00002465,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3658510830136947}
{"item_id":"semeval2015.d000.s006.t000","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d000.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":586,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001381,"input_uncached_usd":0.000029299999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9207997920457274}
{"item_id":"semeval2015.d000.s006.t001","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":830,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00030389999999999996,"input_uncached_usd":0.0000415,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.798218875017483}
{"item_id":"semeval2015.d000.s006.t002","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":4,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"},{"index":5,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d000.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":1937,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008034499999999999,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.0007748,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.480742625019047}
{"item_id":"semeval2015.d000.s007.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031645,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.164923666976392}
{"item_id":"semeval2015.d000.s008.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":619,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026775,"input_uncached_usd":0.00003095,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.815719334001187}
{"item_id":"semeval2015.d000.s008.t001","gold_sense_keys":["treat%2:29:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"treat%2:29:00::","call_ids":["semeval2015.d000.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"treat%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":709,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014424999999999998,"input_uncached_usd":0.00003545,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7385794590227306}
{"item_id":"semeval2015.d000.s008.t002","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013749999999999998,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.75179141695844}
{"item_id":"semeval2015.d000.s008.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016125,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7410875000059605}
{"item_id":"semeval2015.d000.s008.t006","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001612,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.64201554202009}
{"item_id":"semeval2015.d000.s008.t007","gold_sense_keys":["lining%1:06:00::"],"candidates":[{"index":1,"sense_key":"lining%1:06:00::","synset_id":"lining.n.01"},{"index":2,"sense_key":"lining%1:06:01::","synset_id":"liner.n.03"},{"index":3,"sense_key":"lining%1:04:01::","synset_id":"lining.n.03"},{"index":4,"sense_key":"lining%1:04:00::","synset_id":"lining.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"lining%1:06:00::","call_ids":["semeval2015.d000.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"lining%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028544999999999997,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.802536709001288}
{"item_id":"semeval2015.d000.s008.t010","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d000.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.000336,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.121800124994479}
{"item_id":"semeval2015.d000.s009.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"use%2:34:00::","call_ids":["semeval2015.d000.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"use%2:34:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":630,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037109999999999997,"input_uncached_usd":0.0000315,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.846928165992722}
{"item_id":"semeval2015.d000.s009.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002116,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.551007915986702}
{"item_id":"semeval2015.d000.s009.t004","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018744999999999998,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3550019999966025}
{"item_id":"semeval2015.d000.s009.t005","gold_sense_keys":["remove%2:30:00::"],"candidates":[{"index":1,"sense_key":"remove%2:30:00::","synset_id":"remove.v.01"},{"index":2,"sense_key":"remove%2:41:00::","synset_id":"remove.v.02"},{"index":3,"sense_key":"remove%2:40:00::","synset_id":"get_rid_of.v.01"},{"index":4,"sense_key":"remove%2:41:02::","synset_id":"take_out.v.01"},{"index":5,"sense_key":"remove%2:38:00::","synset_id":"remove.v.05"},{"index":6,"sense_key":"remove%2:30:01::","synset_id":"absent.v.01"},{"index":7,"sense_key":"remove%2:41:01::","synset_id":"murder.v.01"},{"index":8,"sense_key":"remove%2:30:02::","synset_id":"remove.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"remove%2:30:00::","call_ids":["semeval2015.d000.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"remove%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027174999999999996,"input_uncached_usd":0.000034949999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.08297395898262}
{"item_id":"semeval2015.d000.s009.t006","gold_sense_keys":["surgery%1:04:01::"],"candidates":[{"index":1,"sense_key":"surgery%1:09:00::","synset_id":"surgery.n.01"},{"index":2,"sense_key":"surgery%1:06:01::","synset_id":"surgery.n.02"},{"index":3,"sense_key":"surgery%1:06:00::","synset_id":"operating_room.n.01"},{"index":4,"sense_key":"surgery%1:04:01::","synset_id":"operation.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"surgery%1:04:01::","call_ids":["semeval2015.d000.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"surgery%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021334999999999998,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.122369083983358}
{"item_id":"semeval2015.d000.s009.t007","gold_sense_keys":["alone%5:00:00:exclusive:00"],"candidates":[{"index":1,"sense_key":"alone%5:00:00:unsocial:00","synset_id":"alone.s.01"},{"index":2,"sense_key":"alone%5:00:00:unaccompanied:00","synset_id":"alone.s.02"},{"index":3,"sense_key":"alone%5:00:00:exclusive:00","synset_id":"alone.s.03"},{"index":4,"sense_key":"alone%5:00:00:incomparable:00","synset_id":"alone.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"alone%5:00:00:exclusive:00","call_ids":["semeval2015.d000.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"alone%5:00:00:exclusive:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011059999999999999,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.12257920805132}
{"item_id":"semeval2015.d000.s009.t009","gold_sense_keys":["spread%2:35:07::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spread%2:35:07::","call_ids":["semeval2015.d000.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spread%2:35:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00053045,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.14157300000079}
{"item_id":"semeval2015.d000.s009.t010","gold_sense_keys":["likely%3:00:00::"],"candidates":[{"index":1,"sense_key":"likely%3:00:00::","synset_id":"likely.a.01"},{"index":2,"sense_key":"likely%3:00:04::","synset_id":"probable.a.01"},{"index":3,"sense_key":"likely%5:00:00:prospective:00","synset_id":"likely.s.01"},{"index":4,"sense_key":"likely%5:00:00:credible:00","synset_id":"likely.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"likely%3:00:04::","call_ids":["semeval2015.d000.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"likely%3:00:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039049999999999995,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.929360791982617}
{"item_id":"semeval2015.d000.s009.t011","gold_sense_keys":["spread%2:35:07::"],"candidates":[{"index":1,"sense_key":"spread%2:35:00::","synset_id":"spread.v.01"},{"index":2,"sense_key":"spread%2:35:07::","synset_id":"spread.v.02"},{"index":3,"sense_key":"spread%2:42:00::","synset_id":"spread.v.03"},{"index":4,"sense_key":"spread%2:35:03::","synset_id":"unfold.v.04"},{"index":5,"sense_key":"spread%2:32:00::","synset_id":"circulate.v.02"},{"index":6,"sense_key":"spread%2:32:02::","synset_id":"go_around.v.02"},{"index":7,"sense_key":"spread%2:38:00::","synset_id":"spread.v.07"},{"index":8,"sense_key":"spread%2:38:02::","synset_id":"diffuse.v.01"},{"index":9,"sense_key":"spread%2:35:12::","synset_id":"spread.v.09"},{"index":10,"sense_key":"spread%2:35:13::","synset_id":"spread.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"spread%2:35:07::","call_ids":["semeval2015.d000.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"spread%2:35:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003513,"input_uncached_usd":0.0000377,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.274216458026785}
{"item_id":"semeval2015.d000.s009.t012","gold_sense_keys":["easily%4:02:01::"],"candidates":[{"index":1,"sense_key":"easily%4:02:01::","synset_id":"easily.r.01"},{"index":2,"sense_key":"easily%4:02:06::","synset_id":"easily.r.02"},{"index":3,"sense_key":"easily%4:02:05::","synset_id":"well.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"easily%4:02:01::","call_ids":["semeval2015.d000.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"easily%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002091,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6689540420193225}
{"item_id":"semeval2015.d000.s009.t013","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d000.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":522,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023769999999999998,"input_uncached_usd":0.000026099999999999997,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9571062499890104}
{"item_id":"semeval2015.d000.s009.t014","gold_sense_keys":["part%1:15:00::","part%1:17:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2015.d000.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":941,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00084705,"input_uncached_usd":0.00004705,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.548613250022754}
{"item_id":"semeval2015.d000.s009.t015","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":797,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045585,"input_uncached_usd":0.00003985,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.5595357500133105}
{"item_id":"semeval2015.d000.s009.t016","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s009.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059135,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.861494500015397}
{"item_id":"semeval2015.d000.s009.t020","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"medicine%1:09:00::","call_ids":["semeval2015.d000.s009.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"medicine%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004932,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.068190041987691}
{"item_id":"semeval2015.d000.s009.t021","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s009.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021299999999999997,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.579796875012107}
{"item_id":"semeval2015.d000.s009.t022","gold_sense_keys":["advanced%5:00:00:late:02"],"candidates":[{"index":1,"sense_key":"advanced%5:00:00:precocious:00","synset_id":"advanced.s.01"},{"index":2,"sense_key":"advanced%5:00:00:late:02","synset_id":"advanced.s.02"},{"index":3,"sense_key":"advanced%5:00:00:progressive:01","synset_id":"advanced.s.03"},{"index":4,"sense_key":"advanced%5:00:02:high:02","synset_id":"advanced.s.04"},{"index":5,"sense_key":"advanced%5:00:00:high-tech:00","synset_id":"advanced.s.05"},{"index":6,"sense_key":"advanced%5:00:00:late:00","synset_id":"advanced.s.06"},{"index":7,"sense_key":"advanced%5:00:00:civilized:00","synset_id":"advanced.s.07"},{"index":8,"sense_key":"advanced%5:00:00:front:00","synset_id":"advance.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"advanced%5:00:00:progressive:01","call_ids":["semeval2015.d000.s009.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"advanced%5:00:00:progressive:01","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047739999999999995,"input_uncached_usd":0.000035799999999999996,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.2342311249813065}
{"item_id":"semeval2015.d000.s009.t024","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s009.t024__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019104999999999998,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.607175790995825}
{"item_id":"semeval2015.d000.s009.t027","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":2,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"},{"index":3,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"affect%2:30:00::","call_ids":["semeval2015.d000.s009.t027__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"affect%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006240000000000001,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.945440584036987}
{"item_id":"semeval2015.d000.s010.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026849999999999997,"input_uncached_usd":0.0000317,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8423829159582965}
{"item_id":"semeval2015.d000.s010.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s010.t002__v1__a1","semeval2015.d000.s010.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":925,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00062225,"input_uncached_usd":0.00004625,"input_cached_usd":0.0,"output_usd":0.000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.578128041001037}
{"item_id":"semeval2015.d000.s010.t004","gold_sense_keys":["treat%2:29:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"treat%2:29:00::","call_ids":["semeval2015.d000.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"treat%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":723,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014494999999999997,"input_uncached_usd":0.00003615,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.525580333021935}
{"item_id":"semeval2015.d000.s010.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s010.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":439,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015634999999999998,"input_uncached_usd":0.000021949999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.441560082952492}
{"item_id":"semeval2015.d000.s010.t007","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s010.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024064999999999997,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.293873500020709}
{"item_id":"semeval2015.d000.s010.t009","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"past%1:10:00::","call_ids":["semeval2015.d000.s010.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"past%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031125,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.596241458028089}
{"item_id":"semeval2015.d000.s011.t000","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s011.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023669999999999998,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9711830830201507}
{"item_id":"semeval2015.d000.s011.t001","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["semeval2015.d000.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031885,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.888080542034004}
{"item_id":"semeval2015.d000.s011.t002","gold_sense_keys":["obtain%2:40:00::"],"candidates":[{"index":1,"sense_key":"obtain%2:40:00::","synset_id":"obtain.v.01"},{"index":2,"sense_key":"obtain%2:30:00::","synset_id":"receive.v.02"},{"index":3,"sense_key":"obtain%2:42:00::","synset_id":"prevail.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"obtain%2:40:00::","call_ids":["semeval2015.d000.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"obtain%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015764999999999998,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.943133749999106}
{"item_id":"semeval2015.d000.s011.t003","gold_sense_keys":["prescription%1:10:01::"],"candidates":[{"index":1,"sense_key":"prescription%1:10:00::","synset_id":"prescription.n.01"},{"index":2,"sense_key":"prescription%1:06:00::","synset_id":"prescription_drug.n.01"},{"index":3,"sense_key":"prescription%1:10:02::","synset_id":"prescription.n.03"},{"index":4,"sense_key":"prescription%1:10:01::","synset_id":"prescription.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"prescription%1:10:01::","call_ids":["semeval2015.d000.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"prescription%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039049999999999995,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.286493833002169}
{"item_id":"semeval2015.d000.s012.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034524999999999996,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.372022083029151}
{"item_id":"semeval2015.d000.s013.t000","gold_sense_keys":["only%4:02:03::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["semeval2015.d000.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00075585,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.555517208005767}
{"item_id":"semeval2015.d000.s013.t001","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2122,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00036849999999999996,"input_uncached_usd":0.00010609999999999999,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.204186957969796}
{"item_id":"semeval2015.d000.s013.t003","gold_sense_keys":["doctor%1:18:00::"],"candidates":[{"index":1,"sense_key":"doctor%1:18:00::","synset_id":"doctor.n.01"},{"index":2,"sense_key":"doctor%1:18:02::","synset_id":"doctor_of_the_church.n.01"},{"index":3,"sense_key":"doctor%1:04:00::","synset_id":"doctor.n.03"},{"index":4,"sense_key":"doctor%1:18:01::","synset_id":"doctor.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"doctor%1:18:00::","call_ids":["semeval2015.d000.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"doctor%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":584,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001892,"input_uncached_usd":0.000029199999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9055156669928692}
{"item_id":"semeval2015.d000.s013.t004","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s013.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002733,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.438312207988929}
{"item_id":"semeval2015.d000.s014.t000","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005958500000000001,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.540864499984309}
{"item_id":"semeval2015.d000.s014.t001","gold_sense_keys":["mg%1:23:00::"],"candidates":[{"index":1,"sense_key":"mg%1:23:00::","synset_id":"milligram.n.01"},{"index":2,"sense_key":"mg%1:27:00::","synset_id":"magnesium.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mg%1:23:00::","call_ids":["semeval2015.d000.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mg%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.000311,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.73447062500054}
{"item_id":"semeval2015.d000.s014.t003","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":803,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037935,"input_uncached_usd":0.00004015,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.840019082999788}
{"item_id":"semeval2015.d000.s014.t005","gold_sense_keys":["calculate%2:31:00::"],"candidates":[{"index":1,"sense_key":"calculate%2:31:00::","synset_id":"calculate.v.01"},{"index":2,"sense_key":"calculate%2:31:03::","synset_id":"calculate.v.02"},{"index":3,"sense_key":"calculate%2:40:00::","synset_id":"account.v.02"},{"index":4,"sense_key":"calculate%2:32:01::","synset_id":"forecast.v.01"},{"index":5,"sense_key":"calculate%2:31:02::","synset_id":"calculate.v.05"},{"index":6,"sense_key":"calculate%2:31:05::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"calculate%2:31:00::","call_ids":["semeval2015.d000.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"calculate%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016265,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0540855419822037}
{"item_id":"semeval2015.d000.s014.t006","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d000.s014.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":636,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004478,"input_uncached_usd":0.0000318,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.834771333029494}
{"item_id":"semeval2015.d000.s014.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s014.t007__v1__a1","semeval2015.d000.s014.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":929,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00044324999999999996,"input_uncached_usd":0.000046450000000000004,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.807667165994644}
{"item_id":"semeval2015.d000.s014.t008","gold_sense_keys":["height%1:07:01::"],"candidates":[{"index":1,"sense_key":"height%1:07:00::","synset_id":"height.n.01"},{"index":2,"sense_key":"height%1:26:00::","synset_id":"acme.n.01"},{"index":3,"sense_key":"height%1:07:01::","synset_id":"stature.n.02"},{"index":4,"sense_key":"height%1:07:02::","synset_id":"altitude.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"height%1:07:01::","call_ids":["semeval2015.d000.s014.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"height%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010989999999999999,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6361784579930827}
{"item_id":"semeval2015.d000.s014.t009","gold_sense_keys":["weight%1:07:00::"],"candidates":[{"index":1,"sense_key":"weight%1:07:00::","synset_id":"weight.n.01"},{"index":2,"sense_key":"weight%1:06:01::","synset_id":"weight.n.02"},{"index":3,"sense_key":"weight%1:07:01::","synset_id":"weight.n.03"},{"index":4,"sense_key":"weight%1:06:00::","synset_id":"weight.n.04"},{"index":5,"sense_key":"weight%1:12:00::","synset_id":"weight.n.05"},{"index":6,"sense_key":"weight%1:23:01::","synset_id":"system_of_weights.n.01"},{"index":7,"sense_key":"weight%1:23:00::","synset_id":"weight_unit.n.02"},{"index":8,"sense_key":"weight%1:23:02::","synset_id":"weight.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"weight%1:07:00::","call_ids":["semeval2015.d000.s014.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"weight%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005767,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.379566957999486}
{"item_id":"semeval2015.d000.s015.t000","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2055,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00033955,"input_uncached_usd":0.00010274999999999999,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.965504083025735}
{"item_id":"semeval2015.d000.s015.t001","gold_sense_keys":["once%4:02:00::"],"candidates":[{"index":1,"sense_key":"once%4:02:00::","synset_id":"once.r.01"},{"index":2,"sense_key":"once%4:02:02::","synset_id":"once.r.02"},{"index":3,"sense_key":"once%4:02:01::","synset_id":"once.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"once%4:02:00::","call_ids":["semeval2015.d000.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"once%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018109999999999998,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2279359999811277}
{"item_id":"semeval2015.d000.s015.t002","gold_sense_keys":["every%5:00:02:all:00"],"candidates":[{"index":1,"sense_key":"every%5:00:00:all:00","synset_id":"every.s.01"},{"index":2,"sense_key":"every%5:00:02:all:00","synset_id":"every.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"every%5:00:02:all:00","call_ids":["semeval2015.d000.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"every%5:00:02:all:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":399,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015434999999999998,"input_uncached_usd":0.00001995,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.854668790998403}
{"item_id":"semeval2015.d000.s015.t003","gold_sense_keys":["week%1:28:00::"],"candidates":[{"index":1,"sense_key":"week%1:28:00::","synset_id":"week.n.01"},{"index":2,"sense_key":"week%1:28:01::","synset_id":"workweek.n.01"},{"index":3,"sense_key":"week%1:28:02::","synset_id":"week.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"week%1:28:00::","call_ids":["semeval2015.d000.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"week%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025755,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4512324999668635}
{"item_id":"semeval2015.d000.s015.t005","gold_sense_keys":["infusion%1:04:00::"],"candidates":[{"index":1,"sense_key":"infusion%1:27:00::","synset_id":"infusion.n.01"},{"index":2,"sense_key":"infusion%1:22:00::","synset_id":"infusion.n.02"},{"index":3,"sense_key":"infusion%1:04:00::","synset_id":"infusion.n.03"},{"index":4,"sense_key":"infusion%1:04:01::","synset_id":"infusion.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"infusion%1:04:00::","call_ids":["semeval2015.d000.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"infusion%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013265,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8065236670081504}
{"item_id":"semeval2015.d000.s015.t006","gold_sense_keys":["last%2:42:00::"],"candidates":[{"index":1,"sense_key":"last%2:42:00::","synset_id":"last.v.01"},{"index":2,"sense_key":"last%2:42:01::","synset_id":"survive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"last%2:42:00::","call_ids":["semeval2015.d000.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"last%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035969999999999996,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.457274082989898}
{"item_id":"semeval2015.d000.s015.t007","gold_sense_keys":["minute%1:28:00::"],"candidates":[{"index":1,"sense_key":"minute%1:28:00::","synset_id":"minute.n.01"},{"index":2,"sense_key":"minute%1:28:01::","synset_id":"moment.n.02"},{"index":3,"sense_key":"minute%1:28:02::","synset_id":"moment.n.01"},{"index":4,"sense_key":"minute%1:23:00::","synset_id":"minute.n.04"},{"index":5,"sense_key":"minute%1:10:00::","synset_id":"minute.n.05"},{"index":6,"sense_key":"minute%1:07:00::","synset_id":"hour.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"minute%1:28:00::","call_ids":["semeval2015.d000.s015.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"minute%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":465,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021344999999999998,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.000186,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8917014590115286}
{"item_id":"semeval2015.d000.s016.t000","gold_sense_keys":["reduce%2:30:00::","reduce%2:30:01::","reduce%2:40:00::"],"candidates":[{"index":1,"sense_key":"reduce%2:30:00::","synset_id":"reduce.v.01"},{"index":2,"sense_key":"reduce%2:30:06::","synset_id":"reduce.v.02"},{"index":3,"sense_key":"reduce%2:41:00::","synset_id":"reduce.v.03"},{"index":4,"sense_key":"reduce%2:40:04::","synset_id":"reduce.v.04"},{"index":5,"sense_key":"reduce%2:37:00::","synset_id":"reduce.v.05"},{"index":6,"sense_key":"reduce%2:30:11::","synset_id":"reduce.v.06"},{"index":7,"sense_key":"reduce%2:30:05::","synset_id":"shrink.v.03"},{"index":8,"sense_key":"reduce%2:40:00::","synset_id":"reduce.v.08"},{"index":9,"sense_key":"reduce%2:30:04::","synset_id":"reduce.v.09"},{"index":10,"sense_key":"reduce%2:30:03::","synset_id":"deoxidize.v.01"},{"index":11,"sense_key":"reduce%2:30:01::","synset_id":"reduce.v.11"},{"index":12,"sense_key":"reduce%2:41:03::","synset_id":"repress.v.01"},{"index":13,"sense_key":"reduce%2:35:00::","synset_id":"reduce.v.13"},{"index":14,"sense_key":"reduce%2:35:01::","synset_id":"reduce.v.14"},{"index":15,"sense_key":"reduce%2:30:12::","synset_id":"reduce.v.15"},{"index":16,"sense_key":"reduce%2:30:08::","synset_id":"abridge.v.01"},{"index":17,"sense_key":"reduce%2:30:10::","synset_id":"boil_down.v.02"},{"index":18,"sense_key":"reduce%2:30:02::","synset_id":"reduce.v.18"},{"index":19,"sense_key":"reduce%2:30:09::","synset_id":"dilute.v.01"},{"index":20,"sense_key":"reduce%2:29:00::","synset_id":"reduce.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"reduce%2:40:00::","call_ids":["semeval2015.d000.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"reduce%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1111,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00026674999999999995,"input_uncached_usd":0.000055549999999999995,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.778103667020332}
{"item_id":"semeval2015.d000.s016.t001","gold_sense_keys":["side_effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"side_effect%1:26:00::","synset_id":"side_effect.n.01"},{"index":2,"sense_key":"side_effect%1:19:00::","synset_id":"side_effect.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"side_effect%1:26:00::","call_ids":["semeval2015.d000.s016.t001__v1__a1","semeval2015.d000.s016.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"side_effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00059225,"input_uncached_usd":0.00004185,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.078148333996069}
{"item_id":"semeval2015.d000.s016.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00066625,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.693533459038008}
{"item_id":"semeval2015.d000.s016.t006","gold_sense_keys":["type%1:09:00::"],"candidates":[{"index":1,"sense_key":"type%1:09:00::","synset_id":"type.n.01"},{"index":2,"sense_key":"type%1:18:00::","synset_id":"character.n.05"},{"index":3,"sense_key":"type%1:14:01::","synset_id":"type.n.03"},{"index":4,"sense_key":"type%1:10:01::","synset_id":"type.n.04"},{"index":5,"sense_key":"type%1:10:00::","synset_id":"type.n.05"},{"index":6,"sense_key":"type%1:06:00::","synset_id":"type.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%1:09:00::","call_ids":["semeval2015.d000.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018734999999999997,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.46626566699706}
{"item_id":"semeval2015.d000.s016.t008","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d000.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00071875,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.992497582978103}
{"item_id":"semeval2015.d000.s016.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020994999999999998,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.474339374981355}
{"item_id":"semeval2015.d000.s017.t000","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2076,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0002382,"input_uncached_usd":0.00010379999999999999,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9604831250035204}
{"item_id":"semeval2015.d000.s017.t001","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.000441,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.3617882920079865}
{"item_id":"semeval2015.d000.s017.t002","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["semeval2015.d000.s017.t002__v1__a1","semeval2015.d000.s017.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":939,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00072535,"input_uncached_usd":0.00004695,"input_cached_usd":0.0,"output_usd":0.0006784,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.21241325000301}
{"item_id":"semeval2015.d000.s017.t004","gold_sense_keys":["fluid%1:27:02::"],"candidates":[{"index":1,"sense_key":"fluid%1:27:02::","synset_id":"fluid.n.01"},{"index":2,"sense_key":"fluid%1:27:00::","synset_id":"fluid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"fluid%1:27:00::","call_ids":["semeval2015.d000.s017.t004__v1__a1","semeval2015.d000.s017.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"fluid%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":837,"cached_input_tokens":0,"output_tokens":2784,"reasoning_output_tokens":2752},"cost":{"currency":"USD","total_usd":0.0011554500000000001,"input_uncached_usd":0.00004185,"input_cached_usd":0.0,"output_usd":0.0011136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.839089207991492}
{"item_id":"semeval2015.d000.s017.t005","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["semeval2015.d000.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004127,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.600873417046387}
{"item_id":"semeval2015.d000.s017.t006","gold_sense_keys":["dehydration%1:26:01::"],"candidates":[{"index":1,"sense_key":"dehydration%1:26:00::","synset_id":"dehydration.n.01"},{"index":2,"sense_key":"dehydration%1:26:01::","synset_id":"dehydration.n.02"},{"index":3,"sense_key":"dehydration%1:22:00::","synset_id":"dehydration.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dehydration%1:26:01::","call_ids":["semeval2015.d000.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dehydration%1:26:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012979999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2844845830113627}
{"item_id":"semeval2015.d000.s017.t008","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d000.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2076,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003918,"input_uncached_usd":0.00010379999999999999,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.706393250031397}
{"item_id":"semeval2015.d000.s017.t009","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"before%4:02:03::","call_ids":["semeval2015.d000.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023234999999999998,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8962946660467423}
{"item_id":"semeval2015.d000.s017.t010","gold_sense_keys":["after%4:02:01::"],"candidates":[{"index":1,"sense_key":"after%4:02:01::","synset_id":"subsequently.r.01"},{"index":2,"sense_key":"after%4:02:00::","synset_id":"after.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"after%4:02:01::","call_ids":["semeval2015.d000.s017.t010__v1__a1","semeval2015.d000.s017.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"after%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":881,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038964999999999996,"input_uncached_usd":0.00004405,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.528102500014938}
{"item_id":"semeval2015.d000.s017.t011","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"dose%1:23:00::","call_ids":["semeval2015.d000.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"dose%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":1553,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006460499999999999,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.0006211999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.68374445894733}
{"item_id":"semeval2015.d000.s018.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036484999999999995,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.401223834021948}
{"item_id":"semeval2015.d000.s018.t001","gold_sense_keys":["delay%2:30:00::"],"candidates":[{"index":1,"sense_key":"delay%2:30:00::","synset_id":"delay.v.01"},{"index":2,"sense_key":"delay%2:42:00::","synset_id":"delay.v.02"},{"index":3,"sense_key":"delay%2:30:01::","synset_id":"stay.v.06"},{"index":4,"sense_key":"delay%2:30:05::","synset_id":"check.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"delay%2:42:00::","call_ids":["semeval2015.d000.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"delay%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026265,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6580407909932546}
{"item_id":"semeval2015.d000.s018.t002","gold_sense_keys":["discontinue%2:42:00::"],"candidates":[{"index":1,"sense_key":"discontinue%2:42:00::","synset_id":"discontinue.v.01"},{"index":2,"sense_key":"discontinue%2:42:01::","synset_id":"discontinue.v.02"},{"index":3,"sense_key":"discontinue%2:30:00::","synset_id":"break.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discontinue%2:42:00::","call_ids":["semeval2015.d000.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discontinue%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018329999999999998,"input_uncached_usd":0.0000233,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5936152079957537}
{"item_id":"semeval2015.d000.s018.t003","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d000.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":508,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.000493,"input_uncached_usd":0.000025399999999999997,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.465229041990824}
{"item_id":"semeval2015.d000.s018.t004","gold_sense_keys":["reduce%2:30:00::"],"candidates":[{"index":1,"sense_key":"reduce%2:30:00::","synset_id":"reduce.v.01"},{"index":2,"sense_key":"reduce%2:30:06::","synset_id":"reduce.v.02"},{"index":3,"sense_key":"reduce%2:41:00::","synset_id":"reduce.v.03"},{"index":4,"sense_key":"reduce%2:40:04::","synset_id":"reduce.v.04"},{"index":5,"sense_key":"reduce%2:37:00::","synset_id":"reduce.v.05"},{"index":6,"sense_key":"reduce%2:30:11::","synset_id":"reduce.v.06"},{"index":7,"sense_key":"reduce%2:30:05::","synset_id":"shrink.v.03"},{"index":8,"sense_key":"reduce%2:40:00::","synset_id":"reduce.v.08"},{"index":9,"sense_key":"reduce%2:30:04::","synset_id":"reduce.v.09"},{"index":10,"sense_key":"reduce%2:30:03::","synset_id":"deoxidize.v.01"},{"index":11,"sense_key":"reduce%2:30:01::","synset_id":"reduce.v.11"},{"index":12,"sense_key":"reduce%2:41:03::","synset_id":"repress.v.01"},{"index":13,"sense_key":"reduce%2:35:00::","synset_id":"reduce.v.13"},{"index":14,"sense_key":"reduce%2:35:01::","synset_id":"reduce.v.14"},{"index":15,"sense_key":"reduce%2:30:12::","synset_id":"reduce.v.15"},{"index":16,"sense_key":"reduce%2:30:08::","synset_id":"abridge.v.01"},{"index":17,"sense_key":"reduce%2:30:10::","synset_id":"boil_down.v.02"},{"index":18,"sense_key":"reduce%2:30:02::","synset_id":"reduce.v.18"},{"index":19,"sense_key":"reduce%2:30:09::","synset_id":"dilute.v.01"},{"index":20,"sense_key":"reduce%2:29:00::","synset_id":"reduce.v.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"reduce%2:30:00::","call_ids":["semeval2015.d000.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"reduce%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1138,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.0001401,"input_uncached_usd":0.0000569,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.371003833017312}
{"item_id":"semeval2015.d000.s018.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004372,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.760062084009405}
{"item_id":"semeval2015.d000.s018.t006","gold_sense_keys":["blood_count%1:23:00::"],"candidates":[{"index":1,"sense_key":"blood_count%1:23:00::","synset_id":"blood_count.n.01"},{"index":2,"sense_key":"blood_count%1:04:00::","synset_id":"blood_count.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"blood_count%1:23:00::","call_ids":["semeval2015.d000.s018.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"blood_count%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003593499999999999,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.534607374982443}
{"item_id":"semeval2015.d000.s018.t007","gold_sense_keys":["abnormal%3:00:00::"],"candidates":[{"index":1,"sense_key":"abnormal%3:00:00::","synset_id":"abnormal.a.01"},{"index":2,"sense_key":"abnormal%3:00:03::","synset_id":"abnormal.a.02"},{"index":3,"sense_key":"abnormal%5:00:00:immoderate:00","synset_id":"abnormal.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"abnormal%3:00:00::","call_ids":["semeval2015.d000.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"abnormal%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":474,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002605,"input_uncached_usd":0.0000237,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.247769374982454}
{"item_id":"semeval2015.d000.s018.t008","gold_sense_keys":["have%2:30:01::","have%2:39:06::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"have%2:39:06::","call_ids":["semeval2015.d000.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"have%2:39:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1173,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00060265,"input_uncached_usd":0.000058649999999999996,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.937173041980714}
{"item_id":"semeval2015.d000.s018.t009","gold_sense_keys":["certain%5:00:00:definite:00"],"candidates":[{"index":1,"sense_key":"certain%5:00:00:definite:00","synset_id":"certain.s.01"},{"index":2,"sense_key":"certain%3:00:02::","synset_id":"certain.a.02"},{"index":3,"sense_key":"certain%3:00:01::","synset_id":"certain.a.03"},{"index":4,"sense_key":"certain%3:00:03::","synset_id":"certain.a.04"},{"index":5,"sense_key":"certain%3:00:05::","synset_id":"sealed.a.01"},{"index":6,"sense_key":"certain%5:00:00:reliable:00","synset_id":"certain.s.02"},{"index":7,"sense_key":"certain%5:00:00:careful:00","synset_id":"certain.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"certain%5:00:00:definite:00","call_ids":["semeval2015.d000.s018.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"certain%5:00:00:definite:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":659,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034655,"input_uncached_usd":0.00003295,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.322540458990261}
{"item_id":"semeval2015.d000.s018.t010","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d000.s018.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018554999999999998,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.704576290969271}
{"item_id":"semeval2015.d000.s018.t011","gold_sense_keys":["side_effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"side_effect%1:26:00::","synset_id":"side_effect.n.01"},{"index":2,"sense_key":"side_effect%1:19:00::","synset_id":"side_effect.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"side_effect%1:26:00::","call_ids":["semeval2015.d000.s018.t011__v1__a1","semeval2015.d000.s018.t011__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"side_effect%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":891,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00074855,"input_uncached_usd":0.00004455,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.01194520801073}
{"item_id":"semeval2015.d000.s019.t002","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%1:04:00::","call_ids":["semeval2015.d000.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":688,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00011759999999999999,"input_uncached_usd":0.000034399999999999996,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.0348730420228094}
{"item_id":"semeval2015.d000.s019.t003","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004369,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.603850832965691}
{"item_id":"semeval2015.d000.s019.t004","gold_sense_keys":["moderate%3:00:00::","moderate%5:00:00:mild:00"],"candidates":[{"index":1,"sense_key":"moderate%3:00:00::","synset_id":"moderate.a.01"},{"index":2,"sense_key":"moderate%5:00:00:mild:00","synset_id":"moderate.s.01"},{"index":3,"sense_key":"moderate%5:00:00:temperate:00","synset_id":"moderate.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"moderate%3:00:00::","call_ids":["semeval2015.d000.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"moderate%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006697999999999999,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.208866541972384}
{"item_id":"semeval2015.d000.s019.t005","gold_sense_keys":["severe%5:00:00:critical:03","severe%5:00:00:intense:00","severe%5:00:01:bad:00"],"candidates":[{"index":1,"sense_key":"severe%5:00:00:intense:00","synset_id":"severe.s.01"},{"index":2,"sense_key":"severe%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":3,"sense_key":"severe%5:00:00:plain:01","synset_id":"austere.s.01"},{"index":4,"sense_key":"severe%5:00:02:nonindulgent:00","synset_id":"severe.s.04"},{"index":5,"sense_key":"severe%5:00:00:critical:03","synset_id":"dangerous.s.01"},{"index":6,"sense_key":"severe%5:00:01:bad:00","synset_id":"severe.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"severe%5:00:01:bad:00","call_ids":["semeval2015.d000.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"severe%5:00:01:bad:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026795,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.183040625008289}
{"item_id":"semeval2015.d000.s019.t007","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2015.d000.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001314,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.792136667005252}
{"item_id":"semeval2015.d000.s020.t000","gold_sense_keys":["more%3:00:01::"],"candidates":[{"index":1,"sense_key":"more%3:00:01::","synset_id":"more.a.01"},{"index":2,"sense_key":"more%3:00:02::","synset_id":"more.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%3:00:01::","call_ids":["semeval2015.d000.s020.t000__v1__a1","semeval2015.d000.s020.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.00067055,"input_uncached_usd":0.000043349999999999997,"input_cached_usd":0.0,"output_usd":0.0006272000000000001,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.881309457996394}
{"item_id":"semeval2015.d000.s020.t001","gold_sense_keys":["information%1:09:00::"],"candidates":[{"index":1,"sense_key":"information%1:10:00::","synset_id":"information.n.01"},{"index":2,"sense_key":"information%1:09:00::","synset_id":"information.n.02"},{"index":3,"sense_key":"information%1:10:02::","synset_id":"information.n.03"},{"index":4,"sense_key":"information%1:14:00::","synset_id":"data.n.01"},{"index":5,"sense_key":"information%1:07:00::","synset_id":"information.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"information%1:09:00::","call_ids":["semeval2015.d000.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"information%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":463,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043914999999999997,"input_uncached_usd":0.00002315,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.323594957997557}
{"item_id":"semeval2015.d000.s020.t002","gold_sense_keys":["see%2:39:13::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"see%2:41:11::","call_ids":["semeval2015.d000.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"see%2:41:11::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1453,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00025824999999999996,"input_uncached_usd":0.00007264999999999999,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.881968958012294}
{"item_id":"semeval2015.d000.s020.t004","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"product%1:27:00::","call_ids":["semeval2015.d000.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"product%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054785,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.737800667004194}
{"item_id":"semeval2015.d000.s020.t006","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"part%1:09:00::","call_ids":["semeval2015.d000.s020.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"part%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":902,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00028189999999999997,"input_uncached_usd":0.0000451,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.42573875002563}
{"item_id":"semeval2015.d000.s022.t000","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":889,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040925,"input_uncached_usd":0.000044449999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.260477999981958}
{"item_id":"semeval2015.d000.s022.t001","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":4,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"},{"index":5,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d000.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":632,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003196,"input_uncached_usd":0.000031599999999999996,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.214185540971812}
{"item_id":"semeval2015.d000.s022.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002114,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7554767499677837}
{"item_id":"semeval2015.d000.s022.t004","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s022.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001862,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.257090208993759}
{"item_id":"semeval2015.d000.s022.t005","gold_sense_keys":["kill%2:35:00::","kill%2:35:01::","kill%2:35:02::"],"candidates":[{"index":1,"sense_key":"kill%2:35:00::","synset_id":"kill.v.01"},{"index":2,"sense_key":"kill%2:41:01::","synset_id":"kill.v.02"},{"index":3,"sense_key":"kill%2:30:08::","synset_id":"stamp_out.v.01"},{"index":4,"sense_key":"kill%2:42:00::","synset_id":"kill.v.04"},{"index":5,"sense_key":"kill%2:39:00::","synset_id":"kill.v.05"},{"index":6,"sense_key":"kill%2:37:00::","synset_id":"kill.v.06"},{"index":7,"sense_key":"kill%2:35:06::","synset_id":"kill.v.07"},{"index":8,"sense_key":"kill%2:35:05::","synset_id":"kill.v.08"},{"index":9,"sense_key":"kill%2:35:02::","synset_id":"kill.v.09"},{"index":10,"sense_key":"kill%2:35:01::","synset_id":"kill.v.10"},{"index":11,"sense_key":"kill%2:34:00::","synset_id":"toss_off.v.02"},{"index":12,"sense_key":"kill%2:30:00::","synset_id":"kill.v.12"},{"index":13,"sense_key":"kill%2:30:04::","synset_id":"kill.v.13"},{"index":14,"sense_key":"kill%2:30:03::","synset_id":"kill.v.14"},{"index":15,"sense_key":"kill%2:30:02::","synset_id":"kill.v.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"kill%2:35:02::","call_ids":["semeval2015.d000.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"kill%2:35:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048754999999999995,"input_uncached_usd":0.00004595,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.985448207997251}
{"item_id":"semeval2015.d000.s022.t006","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s022.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":616,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024199999999999997,"input_uncached_usd":0.000030799999999999996,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.827271500020288}
{"item_id":"semeval2015.d000.s022.t007","gold_sense_keys":["divide%2:35:00::"],"candidates":[{"index":1,"sense_key":"divide%2:41:00::","synset_id":"divide.v.01"},{"index":2,"sense_key":"divide%2:31:00::","synset_id":"divide.v.02"},{"index":3,"sense_key":"divide%2:42:00::","synset_id":"separate.v.01"},{"index":4,"sense_key":"divide%2:35:00::","synset_id":"separate.v.12"},{"index":5,"sense_key":"divide%2:38:00::","synset_id":"separate.v.07"},{"index":6,"sense_key":"divide%2:35:01::","synset_id":"separate.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"divide%2:41:00::","call_ids":["semeval2015.d000.s022.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"divide%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007271000000000001,"input_uncached_usd":0.0000295,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.823852417001035}
{"item_id":"semeval2015.d000.s023.t000","gold_sense_keys":["body%1:08:00::"],"candidates":[{"index":1,"sense_key":"body%1:08:00::","synset_id":"body.n.01"},{"index":2,"sense_key":"body%1:14:00::","synset_id":"body.n.02"},{"index":3,"sense_key":"body%1:08:02::","synset_id":"body.n.03"},{"index":4,"sense_key":"body%1:17:00::","synset_id":"body.n.04"},{"index":5,"sense_key":"body%1:08:01::","synset_id":"torso.n.01"},{"index":6,"sense_key":"body%1:14:02::","synset_id":"body.n.06"},{"index":7,"sense_key":"body%1:07:00::","synset_id":"consistency.n.01"},{"index":8,"sense_key":"body%1:10:00::","synset_id":"body.n.08"},{"index":9,"sense_key":"body%1:07:01::","synset_id":"body.n.09"},{"index":10,"sense_key":"body%1:06:01::","synset_id":"soundbox.n.01"},{"index":11,"sense_key":"body%1:06:00::","synset_id":"body.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"body%1:08:00::","call_ids":["semeval2015.d000.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"body%1:08:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":758,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001723,"input_uncached_usd":0.0000379,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6966143749887124}
{"item_id":"semeval2015.d000.s023.t001","gold_sense_keys":["convert%2:30:03::"],"candidates":[{"index":1,"sense_key":"convert%2:30:00::","synset_id":"convert.v.01"},{"index":2,"sense_key":"convert%2:30:03::","synset_id":"convert.v.02"},{"index":3,"sense_key":"convert%2:30:01::","synset_id":"convert.v.03"},{"index":4,"sense_key":"convert%2:30:02::","synset_id":"change.v.06"},{"index":5,"sense_key":"convert%2:30:07::","synset_id":"convert.v.05"},{"index":6,"sense_key":"convert%2:33:02::","synset_id":"convert.v.06"},{"index":7,"sense_key":"convert%2:33:01::","synset_id":"convert.v.07"},{"index":8,"sense_key":"convert%2:33:00::","synset_id":"convert.v.08"},{"index":9,"sense_key":"convert%2:32:00::","synset_id":"convert.v.09"},{"index":10,"sense_key":"convert%2:30:09::","synset_id":"commute.v.04"},{"index":11,"sense_key":"convert%2:30:04::","synset_id":"convert.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"convert%2:30:04::","call_ids":["semeval2015.d000.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"convert%2:30:04::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019759999999999998,"input_uncached_usd":0.000037199999999999996,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.728763749997597}
{"item_id":"semeval2015.d000.s023.t002","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d000.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":856,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027959999999999997,"input_uncached_usd":0.0000428,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.674635167000815}
{"item_id":"semeval2015.d000.s023.t003","gold_sense_keys":["form%1:07:03::"],"candidates":[{"index":1,"sense_key":"form%1:10:00::","synset_id":"form.n.01"},{"index":2,"sense_key":"form%1:09:01::","synset_id":"kind.n.01"},{"index":3,"sense_key":"form%1:09:00::","synset_id":"form.n.03"},{"index":4,"sense_key":"form%1:07:01::","synset_id":"shape.n.01"},{"index":5,"sense_key":"form%1:08:00::","synset_id":"human_body.n.01"},{"index":6,"sense_key":"form%1:03:00::","synset_id":"shape.n.02"},{"index":7,"sense_key":"form%1:07:00::","synset_id":"form.n.07"},{"index":8,"sense_key":"form%1:10:01::","synset_id":"form.n.08"},{"index":9,"sense_key":"form%1:14:02::","synset_id":"form.n.09"},{"index":10,"sense_key":"form%1:10:02::","synset_id":"form.n.10"},{"index":11,"sense_key":"form%1:07:03::","synset_id":"form.n.11"},{"index":12,"sense_key":"form%1:26:00::","synset_id":"phase.n.02"},{"index":13,"sense_key":"form%1:14:00::","synset_id":"class.n.02"},{"index":14,"sense_key":"form%1:07:02::","synset_id":"form.n.14"},{"index":15,"sense_key":"form%1:06:00::","synset_id":"mannequin.n.02"},{"index":16,"sense_key":"form%1:06:01::","synset_id":"form.n.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"form%1:07:03::","call_ids":["semeval2015.d000.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"form%1:07:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1043,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00031455,"input_uncached_usd":0.00005215,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.637843500007875}
{"item_id":"semeval2015.d000.s023.t004","gold_sense_keys":["block%2:41:01::"],"candidates":[{"index":1,"sense_key":"block%2:35:00::","synset_id":"barricade.v.01"},{"index":2,"sense_key":"block%2:41:00::","synset_id":"obstruct.v.01"},{"index":3,"sense_key":"block%2:41:01::","synset_id":"stop.v.03"},{"index":4,"sense_key":"block%2:32:00::","synset_id":"jam.v.04"},{"index":5,"sense_key":"block%2:41:03::","synset_id":"block.v.05"},{"index":6,"sense_key":"block%2:39:05::","synset_id":"block.v.06"},{"index":7,"sense_key":"block%2:39:00::","synset_id":"obstruct.v.03"},{"index":8,"sense_key":"block%2:35:03::","synset_id":"block.v.08"},{"index":9,"sense_key":"block%2:35:02::","synset_id":"stuff.v.03"},{"index":10,"sense_key":"block%2:35:12::","synset_id":"obstruct.v.02"},{"index":11,"sense_key":"block%2:35:06::","synset_id":"block.v.11"},{"index":12,"sense_key":"block%2:33:00::","synset_id":"parry.v.01"},{"index":13,"sense_key":"block%2:31:00::","synset_id":"forget.v.02"},{"index":14,"sense_key":"block%2:30:01::","synset_id":"block.v.14"},{"index":15,"sense_key":"block%2:30:00::","synset_id":"block.v.15"},{"index":16,"sense_key":"block%2:30:02::","synset_id":"freeze.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"block%2:41:00::","call_ids":["semeval2015.d000.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"block%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":998,"cached_input_tokens":0,"output_tokens":1937,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008246999999999999,"input_uncached_usd":0.0000499,"input_cached_usd":0.0,"output_usd":0.0007748,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.014277749985922}
{"item_id":"semeval2015.d000.s023.t007","gold_sense_keys":["involved%3:00:00::"],"candidates":[{"index":1,"sense_key":"involved%3:00:00::","synset_id":"involved.a.01"},{"index":2,"sense_key":"involved%5:00:00:encumbered:00","synset_id":"involved.s.01"},{"index":3,"sense_key":"involved%5:00:00:attached:01","synset_id":"involved.s.02"},{"index":4,"sense_key":"involved%5:00:00:complex:00","synset_id":"byzantine.s.01"},{"index":5,"sense_key":"involved%5:00:00:enclosed:00","synset_id":"involved.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involved%3:00:00::","call_ids":["semeval2015.d000.s023.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involved%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":549,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034104999999999997,"input_uncached_usd":0.00002745,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.718400583020411}
{"item_id":"semeval2015.d000.s023.t008","gold_sense_keys":["produce%2:36:02::","produce%2:36:03::"],"candidates":[{"index":1,"sense_key":"produce%2:36:02::","synset_id":"produce.v.01"},{"index":2,"sense_key":"produce%2:36:00::","synset_id":"produce.v.02"},{"index":3,"sense_key":"produce%2:36:03::","synset_id":"produce.v.03"},{"index":4,"sense_key":"produce%2:39:01::","synset_id":"produce.v.04"},{"index":5,"sense_key":"produce%2:36:05::","synset_id":"grow.v.07"},{"index":6,"sense_key":"produce%2:39:00::","synset_id":"produce.v.06"},{"index":7,"sense_key":"produce%2:29:00::","synset_id":"grow.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"produce%2:36:00::","call_ids":["semeval2015.d000.s023.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"produce%2:36:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":651,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044855,"input_uncached_usd":0.00003255,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.611550332978368}
{"item_id":"semeval2015.d000.s023.t010","gold_sense_keys":["building_block%1:17:00::"],"candidates":[{"index":1,"sense_key":"building_block%1:17:00::","synset_id":"unit.n.05"},{"index":2,"sense_key":"building_block%1:06:00::","synset_id":"building_block.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"building_block%1:17:00::","call_ids":["semeval2015.d000.s023.t010__v1__a1","semeval2015.d000.s023.t010__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"building_block%1:17:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":807,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043715,"input_uncached_usd":0.00004035,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.830752084031701}
{"item_id":"semeval2015.d000.s023.t013","gold_sense_keys":["genetic%3:01:02::"],"candidates":[{"index":1,"sense_key":"genetic%5:00:00:inheritable:00","synset_id":"familial.s.01"},{"index":2,"sense_key":"genetic%3:01:02::","synset_id":"genic.a.01"},{"index":3,"sense_key":"genetic%3:01:01::","synset_id":"genetic.a.03"},{"index":4,"sense_key":"genetic%3:01:00::","synset_id":"genetic.a.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"genetic%3:01:02::","call_ids":["semeval2015.d000.s023.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"genetic%3:01:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":489,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059405,"input_uncached_usd":0.000024449999999999998,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.788824834045954}
{"item_id":"semeval2015.d000.s023.t014","gold_sense_keys":["material%1:27:00::"],"candidates":[{"index":1,"sense_key":"material%1:27:00::","synset_id":"material.n.01"},{"index":2,"sense_key":"material%1:10:00::","synset_id":"material.n.02"},{"index":3,"sense_key":"material%1:06:00::","synset_id":"fabric.n.01"},{"index":4,"sense_key":"material%1:06:01::","synset_id":"material.n.04"},{"index":5,"sense_key":"material%1:18:00::","synset_id":"material.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"material%1:27:00::","call_ids":["semeval2015.d000.s023.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"material%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":550,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002643,"input_uncached_usd":0.000027499999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.785611125000287}
{"item_id":"semeval2015.d000.s023.t015","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s023.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":582,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047069999999999995,"input_uncached_usd":0.0000291,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.376331333012786}
{"item_id":"semeval2015.d000.s024.t000","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d000.s024.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.000343,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.95477337500779}
{"item_id":"semeval2015.d000.s025.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002874,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.9140401250333525}
{"item_id":"semeval2015.d000.s025.t004","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d000.s025.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":594,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003945,"input_uncached_usd":0.0000297,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.000031000003219}
{"item_id":"semeval2015.d000.s025.t005","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d000.s025.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":465,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015764999999999998,"input_uncached_usd":0.00002325,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.117370707972441}
{"item_id":"semeval2015.d000.s025.t006","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d000.s025.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006836499999999998,"input_uncached_usd":0.00003685,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.101341417001095}
{"item_id":"semeval2015.d000.s025.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s025.t007__v1__a1","semeval2015.d000.s025.t007__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":992,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00044015,"input_uncached_usd":0.000043349999999999997,"input_cached_usd":0.0,"output_usd":0.0003968,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.7389141250168905}
{"item_id":"semeval2015.d000.s025.t012","gold_sense_keys":["before%4:02:03::"],"candidates":[{"index":1,"sense_key":"before%4:02:03::","synset_id":"earlier.r.01"},{"index":2,"sense_key":"before%4:02:01::","synset_id":"ahead.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"before%4:02:03::","call_ids":["semeval2015.d000.s025.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"before%4:02:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025729999999999997,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8747125000227243}
{"item_id":"semeval2015.d000.s026.t000","gold_sense_keys":["effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2015.d000.s026.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":618,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039569999999999997,"input_uncached_usd":0.0000309,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.802720749983564}
{"item_id":"semeval2015.d000.s026.t002","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031435,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.101558417023625}
{"item_id":"semeval2015.d000.s027.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00095665,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.165094624971971}
{"item_id":"semeval2015.d000.s027.t001","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":669,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052625,"input_uncached_usd":0.00003345,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.149712374957744}
{"item_id":"semeval2015.d000.s027.t003","gold_sense_keys":["effect%1:26:00::"],"candidates":[{"index":1,"sense_key":"effect%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"effect%1:07:00::","synset_id":"impression.n.02"},{"index":3,"sense_key":"effect%1:09:00::","synset_id":"effect.n.03"},{"index":4,"sense_key":"effect%1:10:00::","synset_id":"effect.n.04"},{"index":5,"sense_key":"effect%1:07:01::","synset_id":"effect.n.05"},{"index":6,"sense_key":"effect%1:26:00::","synset_id":"effect.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"effect%1:19:00::","call_ids":["semeval2015.d000.s027.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"effect%1:19:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042325,"input_uncached_usd":0.00003285,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.517401416960638}
{"item_id":"semeval2015.d000.s027.t004","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s027.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006747,"input_uncached_usd":0.0000283,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.606207874952815}
{"item_id":"semeval2015.d000.s027.t006","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s027.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039319999999999996,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.826117417018395}
{"item_id":"semeval2015.d000.s027.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d000.s027.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":810,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0003029,"input_uncached_usd":0.000040499999999999995,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.177323542011436}
{"item_id":"semeval2015.d000.s027.t008","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"involve%2:42:00::","call_ids":["semeval2015.d000.s027.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"involve%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008075500000000001,"input_uncached_usd":0.00003315,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.695973124995362}
{"item_id":"semeval2015.d000.s027.t009","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s027.t009__v1__a1","semeval2015.d000.s027.t009__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1013,"cached_input_tokens":0,"output_tokens":1761,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007550500000000001,"input_uncached_usd":0.00005065,"input_cached_usd":0.0,"output_usd":0.0007044,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.023891790944617}
{"item_id":"semeval2015.d000.s027.t010","gold_sense_keys":["locally%4:02:01::"],"candidates":[{"index":1,"sense_key":"locally%4:02:00::","synset_id":"locally.r.01"},{"index":2,"sense_key":"locally%4:02:01::","synset_id":"locally.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"locally%4:02:01::","call_ids":["semeval2015.d000.s027.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"locally%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036239999999999997,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.261626457970124}
{"item_id":"semeval2015.d000.s027.t011","gold_sense_keys":["advanced%5:00:00:late:02"],"candidates":[{"index":1,"sense_key":"advanced%5:00:00:precocious:00","synset_id":"advanced.s.01"},{"index":2,"sense_key":"advanced%5:00:00:late:02","synset_id":"advanced.s.02"},{"index":3,"sense_key":"advanced%5:00:00:progressive:01","synset_id":"advanced.s.03"},{"index":4,"sense_key":"advanced%5:00:02:high:02","synset_id":"advanced.s.04"},{"index":5,"sense_key":"advanced%5:00:00:high-tech:00","synset_id":"advanced.s.05"},{"index":6,"sense_key":"advanced%5:00:00:late:00","synset_id":"advanced.s.06"},{"index":7,"sense_key":"advanced%5:00:00:civilized:00","synset_id":"advanced.s.07"},{"index":8,"sense_key":"advanced%5:00:00:front:00","synset_id":"advance.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"advanced%5:00:00:late:02","call_ids":["semeval2015.d000.s027.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"advanced%5:00:00:late:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":764,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001726,"input_uncached_usd":0.0000382,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.273179874988273}
{"item_id":"semeval2015.d000.s027.t016","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s027.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021104999999999998,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9676496669999324}
{"item_id":"semeval2015.d000.s028.t001","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s028.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":2321,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009554499999999999,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.0009283999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.77895341598196}
{"item_id":"semeval2015.d000.s028.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d000.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":544,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.000648,"input_uncached_usd":0.000027199999999999997,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.622616000007838}
{"item_id":"semeval2015.d000.s028.t005","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d000.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025045,"input_uncached_usd":0.00003925,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.535506415995769}
{"item_id":"semeval2015.d000.s028.t006","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d000.s028.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":638,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005759,"input_uncached_usd":0.000031899999999999996,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.351023124996573}
{"item_id":"semeval2015.d000.s028.t007","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s028.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":458,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006437,"input_uncached_usd":0.000022899999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.79784741596086}
{"item_id":"semeval2015.d000.s028.t012","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s028.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002866,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.545912290981505}
{"item_id":"semeval2015.d000.s029.t000","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s029.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055605,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.350006041990127}
{"item_id":"semeval2015.d000.s029.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d000.s029.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018404999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.4979416669812053}
{"item_id":"semeval2015.d000.s029.t002","gold_sense_keys":["measure%1:10:03::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"measure%1:10:03::","call_ids":["semeval2015.d000.s029.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"measure%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":2512,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.0010417,"input_uncached_usd":0.000036899999999999996,"input_cached_usd":0.0,"output_usd":0.0010048,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.772969375015236}
{"item_id":"semeval2015.d000.s029.t004","gold_sense_keys":["long%4:02:00::"],"candidates":[{"index":1,"sense_key":"long%4:02:00::","synset_id":"long.r.01"},{"index":2,"sense_key":"long%4:02:03::","synset_id":"long.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%4:02:00::","call_ids":["semeval2015.d000.s029.t004__v1__a1","semeval2015.d000.s029.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":841,"cached_input_tokens":0,"output_tokens":1248,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00054125,"input_uncached_usd":0.00004204999999999999,"input_cached_usd":0.0,"output_usd":0.0004992,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.281341874971986}
{"item_id":"semeval2015.d000.s029.t005","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:10:00::","call_ids":["semeval2015.d000.s029.t005__v1__a1","semeval2015.d000.s029.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":2272,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.00095375,"input_uncached_usd":0.000044949999999999995,"input_cached_usd":0.0,"output_usd":0.0009088,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.203042917069979}
{"item_id":"semeval2015.d000.s029.t006","gold_sense_keys":["survive%2:42:01::"],"candidates":[{"index":1,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"survive%2:42:02::","call_ids":["semeval2015.d000.s029.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"survive%2:42:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004437,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.861820999998599}
{"item_id":"semeval2015.d000.s030.t000","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d000.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003098,"input_uncached_usd":0.000021799999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.278064582962543}
{"item_id":"semeval2015.d000.s030.t001","gold_sense_keys":["show%2:32:00::","show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"show%2:32:00::","call_ids":["semeval2015.d000.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"show%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":918,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005131,"input_uncached_usd":0.0000459,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.309522083960474}
{"item_id":"semeval2015.d000.s030.t002","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d000.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":762,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006077000000000001,"input_uncached_usd":0.0000381,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.454084625001997}
{"item_id":"semeval2015.d000.s031.t000","gold_sense_keys":["increase%2:30:02::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"increase%2:30:02::","call_ids":["semeval2015.d000.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"increase%2:30:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053945,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.879155249975156}
{"item_id":"semeval2015.d000.s031.t001","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s031.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":443,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004893499999999999,"input_uncached_usd":0.00002215,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.644943082996178}
{"item_id":"semeval2015.d000.s031.t002","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s031.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055605,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.380141625006218}
{"item_id":"semeval2015.d000.s031.t003","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"patient%1:10:00::","call_ids":["semeval2015.d000.s031.t003__v1__a1","semeval2015.d000.s031.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"patient%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":917,"cached_input_tokens":0,"output_tokens":2848,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.00118505,"input_uncached_usd":0.00004585,"input_cached_usd":0.0,"output_usd":0.0011392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.596396542037837}
{"item_id":"semeval2015.d000.s032.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s032.t000__v1__a1","semeval2015.d000.s032.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":919,"cached_input_tokens":0,"output_tokens":1184,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005195499999999999,"input_uncached_usd":0.00004594999999999999,"input_cached_usd":0.0,"output_usd":0.00047359999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.018967207986861}
{"item_id":"semeval2015.d000.s032.t002","gold_sense_keys":["survive%2:42:01::"],"candidates":[{"index":1,"sense_key":"survive%2:42:01::","synset_id":"survive.v.01"},{"index":2,"sense_key":"survive%2:42:02::","synset_id":"survive.v.02"},{"index":3,"sense_key":"survive%2:42:00::","synset_id":"exist.v.02"},{"index":4,"sense_key":"survive%2:42:03::","synset_id":"outlive.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"survive%2:42:02::","call_ids":["semeval2015.d000.s032.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"survive%2:42:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":565,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029065,"input_uncached_usd":0.00002825,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8389358340064064}
{"item_id":"semeval2015.d000.s032.t003","gold_sense_keys":["average%1:09:00::"],"candidates":[{"index":1,"sense_key":"average%1:09:00::","synset_id":"average.n.01"},{"index":2,"sense_key":"average%1:24:00::","synset_id":"average.n.02"},{"index":3,"sense_key":"average%1:09:01::","synset_id":"average.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%1:09:00::","call_ids":["semeval2015.d000.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001316,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.822522707981989}
{"item_id":"semeval2015.d000.s032.t004","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s032.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003348,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.96357037499547}
{"item_id":"semeval2015.d000.s032.t005","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s032.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023715,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7698923340067267}
{"item_id":"semeval2015.d000.s032.t006","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s032.t006__v1__a1","semeval2015.d000.s032.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00049285,"input_uncached_usd":0.00004484999999999999,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.166184957954101}
{"item_id":"semeval2015.d000.s033.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d000.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039039999999999995,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.189553083037026}
{"item_id":"semeval2015.d000.s033.t001","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004626999999999999,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.210057125019375}
{"item_id":"semeval2015.d000.s033.t002","gold_sense_keys":["cell%1:03:00::"],"candidates":[{"index":1,"sense_key":"cell%1:06:03::","synset_id":"cell.n.01"},{"index":2,"sense_key":"cell%1:03:00::","synset_id":"cell.n.02"},{"index":3,"sense_key":"cell%1:06:00::","synset_id":"cell.n.03"},{"index":4,"sense_key":"cell%1:14:00::","synset_id":"cell.n.04"},{"index":5,"sense_key":"cell%1:06:04::","synset_id":"cellular_telephone.n.01"},{"index":6,"sense_key":"cell%1:06:02::","synset_id":"cell.n.06"},{"index":7,"sense_key":"cell%1:06:01::","synset_id":"cell.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cell%1:03:00::","call_ids":["semeval2015.d000.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cell%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":608,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002672,"input_uncached_usd":0.0000304,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.892939874960575}
{"item_id":"semeval2015.d000.s033.t005","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00","effective%5:00:00:efficacious:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effective%5:00:00:efficacious:00","call_ids":["semeval2015.d000.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effective%5:00:00:efficacious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":656,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002696,"input_uncached_usd":0.0000328,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.918168083007913}
{"item_id":"semeval2015.d000.s034.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":401,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048724999999999994,"input_uncached_usd":0.00002005,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.920594041002914}
{"item_id":"semeval2015.d000.s034.t003","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020695,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6565222089993767}
{"item_id":"semeval2015.d000.s034.t004","gold_sense_keys":["average%5:00:01:normal:01","average%5:00:02:normal:01","average%5:00:03:normal:01"],"candidates":[{"index":1,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":2,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":3,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":6,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%5:00:02:normal:01","call_ids":["semeval2015.d000.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%5:00:02:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024465,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4970854580169544}
{"item_id":"semeval2015.d000.s034.t005","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00066685,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.548638208012562}
{"item_id":"semeval2015.d000.s034.t006","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":719,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052875,"input_uncached_usd":0.00003595,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.648878166975919}
{"item_id":"semeval2015.d000.s034.t007","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003075,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.15533212502487}
{"item_id":"semeval2015.d000.s034.t008","gold_sense_keys":["compare%2:31:00::"],"candidates":[{"index":1,"sense_key":"compare%2:31:00::","synset_id":"compare.v.01"},{"index":2,"sense_key":"compare%2:42:00::","synset_id":"compare.v.02"},{"index":3,"sense_key":"compare%2:31:01::","synset_id":"compare.v.03"},{"index":4,"sense_key":"compare%2:32:00::","synset_id":"compare.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"compare%2:31:00::","call_ids":["semeval2015.d000.s034.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"compare%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033779999999999997,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.441709791019093}
{"item_id":"semeval2015.d000.s034.t009","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s034.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002051,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4214515420026146}
{"item_id":"semeval2015.d000.s035.t000","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s035.t000__v1__a1","semeval2015.d000.s035.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":879,"cached_input_tokens":0,"output_tokens":2529,"reasoning_output_tokens":2496},"cost":{"currency":"USD","total_usd":0.00105555,"input_uncached_usd":0.00004395,"input_cached_usd":0.0,"output_usd":0.0010115999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.061245958029758}
{"item_id":"semeval2015.d000.s035.t004","gold_sense_keys":["past%1:28:00::"],"candidates":[{"index":1,"sense_key":"past%1:28:00::","synset_id":"past.n.01"},{"index":2,"sense_key":"past%1:28:01::","synset_id":"past.n.02"},{"index":3,"sense_key":"past%1:10:00::","synset_id":"past.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"past%1:28:00::","call_ids":["semeval2015.d000.s035.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"past%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003101,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.799461166956462}
{"item_id":"semeval2015.d000.s035.t005","gold_sense_keys":["average%5:00:01:normal:01","average%5:00:02:normal:01","average%5:00:03:normal:01"],"candidates":[{"index":1,"sense_key":"average%5:00:02:normal:01","synset_id":"average.s.01"},{"index":2,"sense_key":"average%5:00:00:common:01","synset_id":"average.s.02"},{"index":3,"sense_key":"average%5:00:00:ordinary:00","synset_id":"average.s.03"},{"index":4,"sense_key":"average%5:00:00:moderate:00","synset_id":"average.s.04"},{"index":5,"sense_key":"average%5:00:03:normal:01","synset_id":"modal.s.01"},{"index":6,"sense_key":"average%5:00:01:normal:01","synset_id":"median.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"average%5:00:02:normal:01","call_ids":["semeval2015.d000.s035.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"average%5:00:02:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":676,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037299999999999996,"input_uncached_usd":0.0000338,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.767501166963484}
{"item_id":"semeval2015.d000.s035.t006","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s035.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0004883999999999999,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.599894292012323}
{"item_id":"semeval2015.d000.s035.t007","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s035.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":734,"cached_input_tokens":0,"output_tokens":3216,"reasoning_output_tokens":3200},"cost":{"currency":"USD","total_usd":0.0013230999999999998,"input_uncached_usd":0.0000367,"input_cached_usd":0.0,"output_usd":0.0012864,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":21.512580042006448}
{"item_id":"semeval2015.d000.s035.t008","gold_sense_keys":["month%1:28:00::"],"candidates":[{"index":1,"sense_key":"month%1:28:01::","synset_id":"calendar_month.n.01"},{"index":2,"sense_key":"month%1:28:00::","synset_id":"month.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"month%1:28:00::","call_ids":["semeval2015.d000.s035.t008__v1__a1","semeval2015.d000.s035.t008__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"month%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":857,"cached_input_tokens":0,"output_tokens":1760,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007468500000000001,"input_uncached_usd":0.00004285,"input_cached_usd":0.0,"output_usd":0.000704,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.536463748954702}
{"item_id":"semeval2015.d000.s035.t009","gold_sense_keys":["group%1:03:00::"],"candidates":[{"index":1,"sense_key":"group%1:03:00::","synset_id":"group.n.01"},{"index":2,"sense_key":"group%1:27:00::","synset_id":"group.n.02"},{"index":3,"sense_key":"group%1:09:00::","synset_id":"group.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"group%1:03:00::","call_ids":["semeval2015.d000.s035.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"group%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025909999999999996,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.4134085410041735}
{"item_id":"semeval2015.d000.s036.t000","gold_sense_keys":["however%4:02:04::"],"candidates":[{"index":1,"sense_key":"however%4:02:00::","synset_id":"however.r.01"},{"index":2,"sense_key":"however%4:02:04::","synset_id":"however.r.02"},{"index":3,"sense_key":"however%4:02:02::","synset_id":"however.r.03"},{"index":4,"sense_key":"however%4:02:01::","synset_id":"however.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"however%4:02:04::","call_ids":["semeval2015.d000.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"however%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":548,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046899999999999996,"input_uncached_usd":0.0000274,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.475583041028585}
{"item_id":"semeval2015.d000.s036.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d000.s036.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":1105,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047969999999999995,"input_uncached_usd":0.0000377,"input_cached_usd":0.0,"output_usd":0.00044199999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.133624874986708}
{"item_id":"semeval2015.d000.s036.t002","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s036.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":427,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036054999999999995,"input_uncached_usd":0.00002135,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.395524583000224}
{"item_id":"semeval2015.d000.s036.t003","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s036.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021265,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.373939041979611}
{"item_id":"semeval2015.d000.s036.t005","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":2,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"},{"index":3,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"affect%2:30:00::","call_ids":["semeval2015.d000.s036.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"affect%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":2257,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009312,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0009027999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.631139124976471}
{"item_id":"semeval2015.d000.s036.t008","gold_sense_keys":["long%3:00:02::"],"candidates":[{"index":1,"sense_key":"long%3:00:02::","synset_id":"long.a.01"},{"index":2,"sense_key":"long%3:00:01::","synset_id":"long.a.02"},{"index":3,"sense_key":"long%5:00:00:tall:00","synset_id":"long.s.01"},{"index":4,"sense_key":"long%3:00:00::","synset_id":"retentive.a.01"},{"index":5,"sense_key":"long%3:00:05::","synset_id":"long.a.05"},{"index":6,"sense_key":"long%3:00:04::","synset_id":"long.a.06"},{"index":7,"sense_key":"long%5:00:00:unsound:00","synset_id":"long.s.02"},{"index":8,"sense_key":"long%5:00:00:provident:00","synset_id":"farseeing.s.02"},{"index":9,"sense_key":"long%5:00:00:abundant:00","synset_id":"long.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"long%3:00:02::","call_ids":["semeval2015.d000.s036.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"long%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":755,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017214999999999998,"input_uncached_usd":0.000037749999999999996,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.681039375020191}
{"item_id":"semeval2015.d000.s036.t009","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s036.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":435,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046334999999999996,"input_uncached_usd":0.00002175,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.279805750004016}
{"item_id":"semeval2015.d000.s036.t010","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"time%1:28:05::","call_ids":["semeval2015.d000.s036.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"time%1:28:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040205,"input_uncached_usd":0.00003725,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.284870958013926}
{"item_id":"semeval2015.d000.s037.t000","gold_sense_keys":["contrast%1:24:00::"],"candidates":[{"index":1,"sense_key":"contrast%1:24:00::","synset_id":"contrast.n.01"},{"index":2,"sense_key":"contrast%1:04:00::","synset_id":"contrast.n.02"},{"index":3,"sense_key":"contrast%1:09:01::","synset_id":"line.n.29"},{"index":4,"sense_key":"contrast%1:09:00::","synset_id":"contrast.n.04"},{"index":5,"sense_key":"contrast%1:07:00::","synset_id":"contrast.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contrast%1:24:00::","call_ids":["semeval2015.d000.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contrast%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":506,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003133,"input_uncached_usd":0.0000253,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.269266207993496}
{"item_id":"semeval2015.d000.s037.t001","gold_sense_keys":["patient%1:18:00::"],"candidates":[{"index":1,"sense_key":"patient%1:18:00::","synset_id":"patient.n.01"},{"index":2,"sense_key":"patient%1:10:00::","synset_id":"affected_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"patient%1:18:00::","call_ids":["semeval2015.d000.s037.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"patient%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006925,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.254861875029746}
{"item_id":"semeval2015.d000.s037.t002","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d000.s037.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003142,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.1741134590120055}
{"item_id":"semeval2015.d000.s037.t003","gold_sense_keys":["affect%2:29:00::"],"candidates":[{"index":1,"sense_key":"affect%2:30:00::","synset_id":"affect.v.01"},{"index":2,"sense_key":"affect%2:29:00::","synset_id":"affect.v.02"},{"index":3,"sense_key":"affect%2:42:00::","synset_id":"involve.v.01"},{"index":4,"sense_key":"affect%2:32:00::","synset_id":"feign.v.01"},{"index":5,"sense_key":"affect%2:37:00::","synset_id":"affect.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"affect%2:30:00::","call_ids":["semeval2015.d000.s037.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"affect%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046914999999999994,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.08555812499253}
{"item_id":"semeval2015.d000.s037.t006","gold_sense_keys":["short%3:00:02::"],"candidates":[{"index":1,"sense_key":"short%3:00:02::","synset_id":"short.a.01"},{"index":2,"sense_key":"short%3:00:01::","synset_id":"short.a.02"},{"index":3,"sense_key":"short%3:00:03::","synset_id":"short.a.03"},{"index":4,"sense_key":"short%5:00:00:insufficient:00","synset_id":"inadequate.s.01"},{"index":5,"sense_key":"short%3:00:00::","synset_id":"unretentive.a.01"},{"index":6,"sense_key":"short%3:00:05::","synset_id":"short.a.06"},{"index":7,"sense_key":"short%3:00:04::","synset_id":"short.a.07"},{"index":8,"sense_key":"short%5:00:02:insufficient:00","synset_id":"light.s.10"},{"index":9,"sense_key":"short%5:00:00:improvident:00","synset_id":"short.s.03"},{"index":10,"sense_key":"short%5:00:00:breakable:00","synset_id":"short.s.04"},{"index":11,"sense_key":"short%5:00:00:discourteous:00","synset_id":"brusque.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"short%3:00:02::","call_ids":["semeval2015.d000.s037.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"short%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":834,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001761,"input_uncached_usd":0.0000417,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.286282957997173}
{"item_id":"semeval2015.d000.s037.t007","gold_sense_keys":["survival%1:26:00::"],"candidates":[{"index":1,"sense_key":"survival%1:26:00::","synset_id":"survival.n.01"},{"index":2,"sense_key":"survival%1:22:00::","synset_id":"survival.n.02"},{"index":3,"sense_key":"survival%1:04:00::","synset_id":"survival.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"survival%1:26:00::","call_ids":["semeval2015.d000.s037.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"survival%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003857,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.714093625021633}
{"item_id":"semeval2015.d000.s037.t008","gold_sense_keys":["time%1:28:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"time%1:28:00::","call_ids":["semeval2015.d000.s037.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"time%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003244,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.841105958039407}
{"item_id":"semeval2015.d001.s000.t000","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"mathematical%3:01:00::","call_ids":["semeval2015.d001.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"mathematical%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":393,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015404999999999998,"input_uncached_usd":0.00001965,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.733788916026242}
{"item_id":"semeval2015.d001.s000.t001","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d001.s000.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":306,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002781,"input_uncached_usd":0.0000153,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.306417458981741}
{"item_id":"semeval2015.d001.s001.t001","gold_sense_keys":["capable%3:00:00::"],"candidates":[{"index":1,"sense_key":"capable%3:00:00::","synset_id":"capable.a.01"},{"index":2,"sense_key":"capable%5:00:00:susceptible:00","synset_id":"capable.s.01"},{"index":3,"sense_key":"capable%3:00:02::","synset_id":"capable.a.03"},{"index":4,"sense_key":"capable%5:00:00:adequate:00","synset_id":"adequate_to.s.01"},{"index":5,"sense_key":"capable%5:00:00:competent:00","synset_id":"able.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"capable%3:00:00::","call_ids":["semeval2015.d001.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"capable%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":430,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002583,"input_uncached_usd":0.000021499999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.458505209011491}
{"item_id":"semeval2015.d001.s001.t002","gold_sense_keys":["make%2:36:12::"],"candidates":[{"index":1,"sense_key":"make%2:41:00::","synset_id":"make.v.01"},{"index":2,"sense_key":"make%2:30:00::","synset_id":"make.v.02"},{"index":3,"sense_key":"make%2:36:00::","synset_id":"make.v.03"},{"index":4,"sense_key":"make%2:32:00::","synset_id":"induce.v.02"},{"index":5,"sense_key":"make%2:36:08::","synset_id":"cause.v.01"},{"index":6,"sense_key":"make%2:36:01::","synset_id":"produce.v.02"},{"index":7,"sense_key":"make%2:31:00::","synset_id":"draw.v.04"},{"index":8,"sense_key":"make%2:36:15::","synset_id":"make.v.08"},{"index":9,"sense_key":"make%2:36:09::","synset_id":"create.v.05"},{"index":10,"sense_key":"make%2:40:01::","synset_id":"gain.v.08"},{"index":11,"sense_key":"make%2:36:05::","synset_id":"do.v.08"},{"index":12,"sense_key":"make%2:42:05::","synset_id":"form.v.02"},{"index":13,"sense_key":"make%2:38:02::","synset_id":"reach.v.07"},{"index":14,"sense_key":"make%2:42:00::","synset_id":"make.v.14"},{"index":15,"sense_key":"make%2:36:11::","synset_id":"make.v.15"},{"index":16,"sense_key":"make%2:36:12::","synset_id":"make.v.16"},{"index":17,"sense_key":"make%2:36:13::","synset_id":"construct.v.01"},{"index":18,"sense_key":"make%2:30:15::","synset_id":"make.v.18"},{"index":19,"sense_key":"make%2:29:08::","synset_id":"make.v.19"},{"index":20,"sense_key":"make%2:41:03::","synset_id":"name.v.03"},{"index":21,"sense_key":"make%2:40:02::","synset_id":"have.v.17"},{"index":22,"sense_key":"make%2:38:00::","synset_id":"reach.v.01"},{"index":23,"sense_key":"make%2:31:13::","synset_id":"lay_down.v.01"},{"index":24,"sense_key":"make%2:41:13::","synset_id":"make.v.24"},{"index":25,"sense_key":"make%2:41:01::","synset_id":"make.v.25"},{"index":26,"sense_key":"make%2:36:04::","synset_id":"hold.v.03"},{"index":27,"sense_key":"make%2:30:02::","synset_id":"make.v.27"},{"index":28,"sense_key":"make%2:38:05::","synset_id":"take.v.27"},{"index":29,"sense_key":"make%2:29:00::","synset_id":"stool.v.04"},{"index":30,"sense_key":"make%2:42:09::","synset_id":"make.v.30"},{"index":31,"sense_key":"make%2:42:13::","synset_id":"make.v.31"},{"index":32,"sense_key":"make%2:42:12::","synset_id":"make.v.32"},{"index":33,"sense_key":"make%2:42:11::","synset_id":"make.v.33"},{"index":34,"sense_key":"make%2:42:15::","synset_id":"make.v.34"},{"index":35,"sense_key":"make%2:39:12::","synset_id":"make.v.35"},{"index":36,"sense_key":"make%2:38:10::","synset_id":"make.v.36"},{"index":37,"sense_key":"make%2:38:15::","synset_id":"make.v.37"},{"index":38,"sense_key":"make%2:36:10::","synset_id":"make.v.38"},{"index":39,"sense_key":"make%2:36:07::","synset_id":"cook.v.02"},{"index":40,"sense_key":"make%2:35:13::","synset_id":"seduce.v.01"},{"index":41,"sense_key":"make%2:32:14::","synset_id":"make.v.41"},{"index":42,"sense_key":"make%2:32:09::","synset_id":"make.v.42"},{"index":43,"sense_key":"make%2:31:10::","synset_id":"make.v.43"},{"index":44,"sense_key":"make%2:31:12::","synset_id":"make.v.44"},{"index":45,"sense_key":"make%2:30:10::","synset_id":"make.v.45"},{"index":46,"sense_key":"make%2:30:14::","synset_id":"make.v.46"},{"index":47,"sense_key":"make%2:30:12::","synset_id":"make.v.47"},{"index":48,"sense_key":"make%2:29:06::","synset_id":"make.v.48"},{"index":49,"sense_key":"make%2:29:01::","synset_id":"make.v.49"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":16,"chosen_sense_key":"make%2:36:12::","call_ids":["semeval2015.d001.s001.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":16,"predicted_sense_key":"make%2:36:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2162,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002937,"input_uncached_usd":0.0001081,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.741738374985289}
{"item_id":"semeval2015.d001.s001.t003","gold_sense_keys":["simple%3:00:02::","simple%5:00:00:easy:01"],"candidates":[{"index":1,"sense_key":"simple%3:00:02::","synset_id":"simple.a.01"},{"index":2,"sense_key":"simple%5:00:00:easy:01","synset_id":"elementary.s.01"},{"index":3,"sense_key":"simple%5:00:02:plain:01","synset_id":"bare.s.04"},{"index":4,"sense_key":"simple%5:00:00:naive:00","synset_id":"childlike.s.02"},{"index":5,"sense_key":"simple%5:00:00:retarded:00","synset_id":"dim-witted.s.01"},{"index":6,"sense_key":"simple%3:00:01::","synset_id":"simple.a.06"},{"index":7,"sense_key":"simple%5:00:00:plain:01","synset_id":"simple.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"simple%5:00:00:easy:01","call_ids":["semeval2015.d001.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"simple%5:00:00:easy:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023669999999999998,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.423939959029667}
{"item_id":"semeval2015.d001.s001.t004","gold_sense_keys":["operation%1:04:01::"],"candidates":[{"index":1,"sense_key":"operation%1:26:00::","synset_id":"operation.n.01"},{"index":2,"sense_key":"operation%1:04:06::","synset_id":"operation.n.02"},{"index":3,"sense_key":"operation%1:04:03::","synset_id":"operation.n.03"},{"index":4,"sense_key":"operation%1:22:00::","synset_id":"operation.n.04"},{"index":5,"sense_key":"operation%1:04:02::","synset_id":"operation.n.05"},{"index":6,"sense_key":"operation%1:04:00::","synset_id":"operation.n.06"},{"index":7,"sense_key":"operation%1:04:05::","synset_id":"operation.n.07"},{"index":8,"sense_key":"operation%1:22:01::","synset_id":"operation.n.08"},{"index":9,"sense_key":"operation%1:09:00::","synset_id":"process.n.02"},{"index":10,"sense_key":"operation%1:04:01::","synset_id":"mathematical_process.n.01"},{"index":11,"sense_key":"operation%1:04:07::","synset_id":"operation.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":10,"chosen_sense_key":"operation%1:04:01::","call_ids":["semeval2015.d001.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":10,"predicted_sense_key":"operation%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027505,"input_uncached_usd":0.00003825,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.443973083049059}
{"item_id":"semeval2015.d001.s001.t006","gold_sense_keys":["3d%1:09:00::"],"candidates":[{"index":1,"sense_key":"3-d%1:10:00::","synset_id":"three-d.n.01"},{"index":2,"sense_key":"3-d%1:09:00::","synset_id":"three-d.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"3-d%1:10:00::","call_ids":["semeval2015.d001.s001.t006__v1__a1","semeval2015.d001.s001.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"3-d%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":609,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00060645,"input_uncached_usd":0.000030449999999999998,"input_cached_usd":0.0,"output_usd":0.000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.246583333006129}
{"item_id":"semeval2015.d001.s002.t000","gold_sense_keys":["actually%4:02:00::","actually%4:02:02::"],"candidates":[{"index":1,"sense_key":"actually%4:02:01::","synset_id":"actually.r.01"},{"index":2,"sense_key":"actually%4:02:00::","synset_id":"actually.r.02"},{"index":3,"sense_key":"actually%4:02:03::","synset_id":"actually.r.03"},{"index":4,"sense_key":"actually%4:02:02::","synset_id":"actually.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"actually%4:02:02::","call_ids":["semeval2015.d001.s002.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"actually%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":396,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004102,"input_uncached_usd":0.0000198,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.278749916004017}
{"item_id":"semeval2015.d001.s002.t002","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":273,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00068565,"input_uncached_usd":0.00001365,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.300472291011829}
{"item_id":"semeval2015.d001.s002.t003","gold_sense_keys":["know%2:31:15::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"know%2:31:01::","call_ids":["semeval2015.d001.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"know%2:31:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":730,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032450000000000003,"input_uncached_usd":0.0000365,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.565634750004392}
{"item_id":"semeval2015.d001.s002.t004","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s002.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018485,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3268518749973737}
{"item_id":"semeval2015.d001.s004.t002","gold_sense_keys":["now%4:02:00::","now%4:02:04::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"now%4:02:04::","call_ids":["semeval2015.d001.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"now%4:02:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023879999999999998,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.99262145900866}
{"item_id":"semeval2015.d001.s004.t003","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018624999999999997,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9361645829631016}
{"item_id":"semeval2015.d001.s004.t004","gold_sense_keys":["little%3:00:03::"],"candidates":[{"index":1,"sense_key":"little%3:00:01::","synset_id":"small.a.01"},{"index":2,"sense_key":"little%3:00:03::","synset_id":"little.a.02"},{"index":3,"sense_key":"little%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"little%5:00:00:unimportant:00","synset_id":"fiddling.s.01"},{"index":5,"sense_key":"little%5:00:00:soft:04","synset_id":"little.s.03"},{"index":6,"sense_key":"little%3:00:00::","synset_id":"short.a.03"},{"index":7,"sense_key":"little%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":8,"sense_key":"little%5:00:00:emotional:00","synset_id":"little.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"little%3:00:03::","call_ids":["semeval2015.d001.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"little%3:00:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000217,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.970765082980506}
{"item_id":"semeval2015.d001.s004.t005","gold_sense_keys":["mathematical%3:01:00::"],"candidates":[{"index":1,"sense_key":"mathematical%3:01:00::","synset_id":"mathematical.a.01"},{"index":2,"sense_key":"mathematical%3:00:00::","synset_id":"numerical.a.03"},{"index":3,"sense_key":"mathematical%5:00:00:unquestionable:00","synset_id":"mathematical.s.01"},{"index":4,"sense_key":"mathematical%5:00:00:possible:00","synset_id":"mathematical.s.02"},{"index":5,"sense_key":"mathematical%5:00:00:exact:00","synset_id":"mathematical.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"mathematical%5:00:00:unquestionable:00","call_ids":["semeval2015.d001.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"mathematical%5:00:00:unquestionable:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043824999999999995,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.819732000003569}
{"item_id":"semeval2015.d001.s005.t000","gold_sense_keys":["here%4:02:00::"],"candidates":[{"index":1,"sense_key":"here%4:02:00::","synset_id":"here.r.01"},{"index":2,"sense_key":"here%4:02:02::","synset_id":"here.r.02"},{"index":3,"sense_key":"here%4:02:01::","synset_id":"here.r.03"},{"index":4,"sense_key":"here%4:02:04::","synset_id":"here.r.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"here%4:02:00::","call_ids":["semeval2015.d001.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"here%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002317,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.223293667018879}
{"item_id":"semeval2015.d001.s005.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023065,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.17580425000051}
{"item_id":"semeval2015.d001.s005.t002","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003424,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.46902683301596}
{"item_id":"semeval2015.d001.s006.t000","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"main%5:00:00:intense:00","call_ids":["semeval2015.d001.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"main%5:00:00:intense:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048729999999999997,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.696068416989874}
{"item_id":"semeval2015.d001.s006.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034304999999999996,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.155247124959715}
{"item_id":"semeval2015.d001.s007.t000","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"main%5:00:00:intense:00","call_ids":["semeval2015.d001.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"main%5:00:00:intense:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":391,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035874999999999997,"input_uncached_usd":0.000019549999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.762391791038681}
{"item_id":"semeval2015.d001.s007.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024009999999999998,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.13766308297636}
{"item_id":"semeval2015.d001.s007.t002","gold_sense_keys":["consist%2:42:01::"],"candidates":[{"index":1,"sense_key":"consist%2:42:00::","synset_id":"dwell.v.02"},{"index":2,"sense_key":"consist%2:42:04::","synset_id":"consist.v.02"},{"index":3,"sense_key":"consist%2:42:02::","synset_id":"consist.v.03"},{"index":4,"sense_key":"consist%2:42:01::","synset_id":"consist.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"consist%2:42:01::","call_ids":["semeval2015.d001.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"consist%2:42:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041205,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.678184541000519}
{"item_id":"semeval2015.d001.s007.t003","gold_sense_keys":["console%1:06:02::"],"candidates":[{"index":1,"sense_key":"console%1:06:03::","synset_id":"console_table.n.01"},{"index":2,"sense_key":"console%1:06:02::","synset_id":"console.n.02"},{"index":3,"sense_key":"console%1:06:01::","synset_id":"console.n.03"},{"index":4,"sense_key":"console%1:06:00::","synset_id":"cabinet.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"console%1:06:02::","call_ids":["semeval2015.d001.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"console%1:06:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005384000000000001,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.677247333049309}
{"item_id":"semeval2015.d001.s007.t005","gold_sense_keys":["3d%1:09:00::"],"candidates":[{"index":1,"sense_key":"3-d%1:10:00::","synset_id":"three-d.n.01"},{"index":2,"sense_key":"3-d%1:09:00::","synset_id":"three-d.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"3-d%1:09:00::","call_ids":["semeval2015.d001.s007.t005__v1__a1","semeval2015.d001.s007.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"3-d%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":697,"cached_input_tokens":0,"output_tokens":1632,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006876499999999999,"input_uncached_usd":0.00003484999999999999,"input_cached_usd":0.0,"output_usd":0.0006528,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.936356250022072}
{"item_id":"semeval2015.d001.s008.t000","gold_sense_keys":["below%4:02:01::"],"candidates":[{"index":1,"sense_key":"below%4:02:01::","synset_id":"below.r.01"},{"index":2,"sense_key":"below%4:02:00::","synset_id":"below.r.02"},{"index":3,"sense_key":"below%4:02:02::","synset_id":"below.r.03"},{"index":4,"sense_key":"below%4:02:06::","synset_id":"downstairs.r.01"},{"index":5,"sense_key":"below%4:02:03::","synset_id":"under.r.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"below%4:02:01::","call_ids":["semeval2015.d001.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"below%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018199999999999998,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.985117915959563}
{"item_id":"semeval2015.d001.s008.t001","gold_sense_keys":["find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["semeval2015.d001.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1064,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009812,"input_uncached_usd":0.0000532,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.947609167022165}
{"item_id":"semeval2015.d001.s008.t004","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00048465,"input_uncached_usd":0.00001745,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.572631457995158}
{"item_id":"semeval2015.d001.s008.t005","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006001000000000001,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.309112875023857}
{"item_id":"semeval2015.d001.s008.t006","gold_sense_keys":["do%2:36:01::","do%2:41:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":869,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00043385,"input_uncached_usd":0.00004345,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.386735167005099}
{"item_id":"semeval2015.d001.s008.t007","gold_sense_keys":["calculation%1:04:00::","calculation%1:09:00::"],"candidates":[{"index":1,"sense_key":"calculation%1:04:00::","synset_id":"calculation.n.01"},{"index":2,"sense_key":"calculation%1:09:00::","synset_id":"calculation.n.02"},{"index":3,"sense_key":"calculation%1:09:01::","synset_id":"calculation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"calculation%1:04:00::","call_ids":["semeval2015.d001.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"calculation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":385,"cached_input_tokens":0,"output_tokens":977,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041005,"input_uncached_usd":0.00001925,"input_cached_usd":0.0,"output_usd":0.00039079999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.883893334015738}
{"item_id":"semeval2015.d001.s009.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":477,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033745,"input_uncached_usd":0.00002385,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.8030698749935254}
{"item_id":"semeval2015.d001.s009.t001","gold_sense_keys":["understand%2:31:00::","understand%2:31:01::"],"candidates":[{"index":1,"sense_key":"understand%2:31:00::","synset_id":"understand.v.01"},{"index":2,"sense_key":"understand%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"understand%2:31:03::","synset_id":"understand.v.03"},{"index":4,"sense_key":"understand%2:31:02::","synset_id":"understand.v.04"},{"index":5,"sense_key":"understand%2:31:04::","synset_id":"sympathize.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"understand%2:31:03::","call_ids":["semeval2015.d001.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"understand%2:31:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036419999999999996,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.745879583002534}
{"item_id":"semeval2015.d001.s009.t002","gold_sense_keys":["work%2:35:02::","work%2:41:03::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"work%2:41:03::","call_ids":["semeval2015.d001.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"work%2:41:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1454,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005399,"input_uncached_usd":0.00007269999999999999,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.410440334002487}
{"item_id":"semeval2015.d001.s009.t003","gold_sense_keys":["best%5:00:00:advisable:00"],"candidates":[{"index":1,"sense_key":"best%3:00:00::","synset_id":"best.a.01"},{"index":2,"sense_key":"best%5:00:00:advisable:00","synset_id":"better.s.01"},{"index":3,"sense_key":"good%3:00:01::","synset_id":"good.a.01"},{"index":4,"sense_key":"full%5:00:00:ample:00","synset_id":"full.s.04"},{"index":5,"sense_key":"good%3:00:02::","synset_id":"good.a.03"},{"index":6,"sense_key":"estimable%5:00:00:reputable:00","synset_id":"estimable.s.01"},{"index":7,"sense_key":"beneficial%5:00:00:advantageous:00","synset_id":"beneficial.s.01"},{"index":8,"sense_key":"good%5:00:00:nice:00","synset_id":"good.s.04"},{"index":9,"sense_key":"good%5:00:00:righteous:00","synset_id":"good.s.05"},{"index":10,"sense_key":"adept%5:00:00:skilled:00","synset_id":"adept.s.01"},{"index":11,"sense_key":"good%5:00:00:complete:00","synset_id":"good.s.07"},{"index":12,"sense_key":"dear%5:00:00:close:02","synset_id":"dear.s.02"},{"index":13,"sense_key":"dependable%5:00:00:sound:00","synset_id":"dependable.s.03"},{"index":14,"sense_key":"good%5:00:00:opportune:00","synset_id":"good.s.10"},{"index":15,"sense_key":"good%5:00:00:fortunate:00","synset_id":"good.s.11"},{"index":16,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":17,"sense_key":"good%5:00:00:pleasing:00","synset_id":"good.s.13"},{"index":18,"sense_key":"good%5:00:00:intellectual:00","synset_id":"good.s.14"},{"index":19,"sense_key":"good%5:00:00:healthy:00","synset_id":"good.s.15"},{"index":20,"sense_key":"good%5:00:00:healthful:00","synset_id":"good.s.16"},{"index":21,"sense_key":"good%5:00:00:genuine:00","synset_id":"good.s.17"},{"index":22,"sense_key":"good%5:00:00:fresh:01","synset_id":"good.s.18"},{"index":23,"sense_key":"good%5:00:00:discriminating:00","synset_id":"good.s.19"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"best%5:00:00:advisable:00","call_ids":["semeval2015.d001.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"best%5:00:00:advisable:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1319,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.00089155,"input_uncached_usd":0.00006594999999999999,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.869039500015788}
{"item_id":"semeval2015.d001.s009.t004","gold_sense_keys":["thing%1:04:00::"],"candidates":[{"index":1,"sense_key":"thing%1:26:00::","synset_id":"thing.n.01"},{"index":2,"sense_key":"thing%1:04:00::","synset_id":"thing.n.02"},{"index":3,"sense_key":"thing%1:09:00::","synset_id":"thing.n.03"},{"index":4,"sense_key":"thing%1:06:00::","synset_id":"thing.n.04"},{"index":5,"sense_key":"thing%1:11:00::","synset_id":"thing.n.05"},{"index":6,"sense_key":"thing%1:09:02::","synset_id":"matter.n.01"},{"index":7,"sense_key":"thing%1:10:00::","synset_id":"thing.n.07"},{"index":8,"sense_key":"thing%1:06:01::","synset_id":"thing.n.08"},{"index":9,"sense_key":"thing%1:07:00::","synset_id":"thing.n.09"},{"index":10,"sense_key":"thing%1:09:01::","synset_id":"thing.n.10"},{"index":11,"sense_key":"thing%1:12:00::","synset_id":"thing.n.11"},{"index":12,"sense_key":"thing%1:03:00::","synset_id":"thing.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"thing%1:04:00::","call_ids":["semeval2015.d001.s009.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"thing%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":677,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032225,"input_uncached_usd":0.000033849999999999996,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.909633209055755}
{"item_id":"semeval2015.d001.s009.t005","gold_sense_keys":["do%2:36:01::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":877,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038304999999999996,"input_uncached_usd":0.000043849999999999995,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.19455920899054}
{"item_id":"semeval2015.d001.s009.t006","gold_sense_keys":["realize%2:31:01::"],"candidates":[{"index":1,"sense_key":"realize%2:31:01::","synset_id":"recognize.v.02"},{"index":2,"sense_key":"realize%2:31:00::","synset_id":"understand.v.02"},{"index":3,"sense_key":"realize%2:36:00::","synset_id":"realize.v.03"},{"index":4,"sense_key":"realize%2:40:01::","synset_id":"gain.v.08"},{"index":5,"sense_key":"realize%2:40:00::","synset_id":"realize.v.05"},{"index":6,"sense_key":"realize%2:36:07::","synset_id":"realize.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"realize%2:31:00::","call_ids":["semeval2015.d001.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"realize%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":576,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009056,"input_uncached_usd":0.0000288,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.653998874942772}
{"item_id":"semeval2015.d001.s009.t008","gold_sense_keys":["convert%2:30:00::","convert%2:30:02::","convert%2:30:03::"],"candidates":[{"index":1,"sense_key":"convert%2:30:00::","synset_id":"convert.v.01"},{"index":2,"sense_key":"convert%2:30:03::","synset_id":"convert.v.02"},{"index":3,"sense_key":"convert%2:30:01::","synset_id":"convert.v.03"},{"index":4,"sense_key":"convert%2:30:02::","synset_id":"change.v.06"},{"index":5,"sense_key":"convert%2:30:07::","synset_id":"convert.v.05"},{"index":6,"sense_key":"convert%2:33:02::","synset_id":"convert.v.06"},{"index":7,"sense_key":"convert%2:33:01::","synset_id":"convert.v.07"},{"index":8,"sense_key":"convert%2:33:00::","synset_id":"convert.v.08"},{"index":9,"sense_key":"convert%2:32:00::","synset_id":"convert.v.09"},{"index":10,"sense_key":"convert%2:30:09::","synset_id":"commute.v.04"},{"index":11,"sense_key":"convert%2:30:04::","synset_id":"convert.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"convert%2:30:03::","call_ids":["semeval2015.d001.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"convert%2:30:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":714,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003493,"input_uncached_usd":0.0000357,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.647025000012945}
{"item_id":"semeval2015.d001.s009.t010","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":338,"cached_input_tokens":0,"output_tokens":1361,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005612999999999999,"input_uncached_usd":0.0000169,"input_cached_usd":0.0,"output_usd":0.0005444,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.34611441695597}
{"item_id":"semeval2015.d001.s009.t011","gold_sense_keys":["know%2:31:15::"],"candidates":[{"index":1,"sense_key":"know%2:31:01::","synset_id":"know.v.01"},{"index":2,"sense_key":"know%2:31:03::","synset_id":"know.v.02"},{"index":3,"sense_key":"know%2:31:02::","synset_id":"know.v.03"},{"index":4,"sense_key":"know%2:31:00::","synset_id":"know.v.04"},{"index":5,"sense_key":"know%2:31:04::","synset_id":"know.v.05"},{"index":6,"sense_key":"know%2:31:06::","synset_id":"acknowledge.v.06"},{"index":7,"sense_key":"know%2:31:15::","synset_id":"know.v.07"},{"index":8,"sense_key":"know%2:35:00::","synset_id":"sleep_together.v.01"},{"index":9,"sense_key":"know%2:31:11::","synset_id":"know.v.09"},{"index":10,"sense_key":"know%2:31:12::","synset_id":"know.v.10"},{"index":11,"sense_key":"know%2:31:14::","synset_id":"know.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"know%2:31:00::","call_ids":["semeval2015.d001.s009.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"know%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":795,"cached_input_tokens":0,"output_tokens":2385,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.0009937499999999998,"input_uncached_usd":0.00003975,"input_cached_usd":0.0,"output_usd":0.000954,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.080247499980032}
{"item_id":"semeval2015.d001.s010.t000","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d001.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023119999999999998,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4743544579832815}
{"item_id":"semeval2015.d001.s010.t004","gold_sense_keys":["hard%3:00:06::"],"candidates":[{"index":1,"sense_key":"hard%3:00:06::","synset_id":"difficult.a.01"},{"index":2,"sense_key":"hard%3:00:02::","synset_id":"hard.a.02"},{"index":3,"sense_key":"hard%3:00:01::","synset_id":"hard.a.03"},{"index":4,"sense_key":"hard%5:00:00:strong:00","synset_id":"hard.s.01"},{"index":5,"sense_key":"hard%5:00:01:effortful:00","synset_id":"arduous.s.01"},{"index":6,"sense_key":"hard%3:00:00::","synset_id":"unvoiced.a.01"},{"index":7,"sense_key":"hard%3:00:05::","synset_id":"hard.a.07"},{"index":8,"sense_key":"hard%3:00:03::","synset_id":"hard.a.08"},{"index":9,"sense_key":"hard%5:00:00:indulgent:00","synset_id":"intemperate.s.01"},{"index":10,"sense_key":"hard%5:00:00:alcoholic:00","synset_id":"hard.s.04"},{"index":11,"sense_key":"hard%5:00:00:bad:00","synset_id":"hard.s.05"},{"index":12,"sense_key":"hard%5:00:00:stale:00","synset_id":"hard.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"hard%3:00:06::","call_ids":["semeval2015.d001.s010.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"hard%3:00:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":805,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00020025,"input_uncached_usd":0.000040249999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2895765830180608}
{"item_id":"semeval2015.d001.s011.t001","gold_sense_keys":["closely%4:02:02::"],"candidates":[{"index":1,"sense_key":"closely%4:02:00::","synset_id":"closely.r.01"},{"index":2,"sense_key":"closely%4:02:02::","synset_id":"close.r.02"},{"index":3,"sense_key":"closely%4:02:01::","synset_id":"closely.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"closely%4:02:02::","call_ids":["semeval2015.d001.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"closely%4:02:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003081,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.710422749980353}
{"item_id":"semeval2015.d001.s011.t002","gold_sense_keys":["look%2:39:01::"],"candidates":[{"index":1,"sense_key":"look%2:39:00::","synset_id":"look.v.01"},{"index":2,"sense_key":"look%2:39:01::","synset_id":"look.v.02"},{"index":3,"sense_key":"look%2:29:00::","synset_id":"look.v.03"},{"index":4,"sense_key":"look%2:39:02::","synset_id":"search.v.02"},{"index":5,"sense_key":"look%2:42:00::","synset_id":"front.v.01"},{"index":6,"sense_key":"look%2:41:13::","synset_id":"attend.v.02"},{"index":7,"sense_key":"look%2:32:00::","synset_id":"look.v.07"},{"index":8,"sense_key":"look%2:31:00::","synset_id":"expect.v.03"},{"index":9,"sense_key":"look%2:42:02::","synset_id":"look.v.09"},{"index":10,"sense_key":"look%2:31:02::","synset_id":"count.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"look%2:39:01::","call_ids":["semeval2015.d001.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"look%2:39:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040104999999999996,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.7301541669876315}
{"item_id":"semeval2015.d001.s011.t003","gold_sense_keys":["very%4:02:00::"],"candidates":[{"index":1,"sense_key":"very%4:02:00::","synset_id":"very.r.01"},{"index":2,"sense_key":"very%4:02:01::","synset_id":"very.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"very%4:02:00::","call_ids":["semeval2015.d001.s011.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"very%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033255,"input_uncached_usd":0.00001895,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.180549249984324}
{"item_id":"semeval2015.d001.s011.t004","gold_sense_keys":["similar%3:00:00::","similar%3:00:02::","similar%3:00:04::"],"candidates":[{"index":1,"sense_key":"similar%3:00:00::","synset_id":"similar.a.01"},{"index":2,"sense_key":"similar%3:00:04::","synset_id":"alike.a.01"},{"index":3,"sense_key":"similar%3:00:02::","synset_id":"like.a.01"},{"index":4,"sense_key":"similar%5:00:00:synonymous:00","synset_id":"similar.s.01"},{"index":5,"sense_key":"similar%5:00:00:replaceable:00","synset_id":"exchangeable.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"similar%3:00:02::","call_ids":["semeval2015.d001.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"similar%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046674999999999993,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.762760625046212}
{"item_id":"semeval2015.d001.s012.t000","gold_sense_keys":["addition%1:04:01::"],"candidates":[{"index":1,"sense_key":"addition%1:06:00::","synset_id":"addition.n.01"},{"index":2,"sense_key":"addition%1:04:02::","synset_id":"addition.n.02"},{"index":3,"sense_key":"addition%1:23:00::","synset_id":"addition.n.03"},{"index":4,"sense_key":"addition%1:21:00::","synset_id":"accession.n.03"},{"index":5,"sense_key":"addition%1:15:00::","synset_id":"addition.n.05"},{"index":6,"sense_key":"addition%1:04:01::","synset_id":"summation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"addition%1:04:01::","call_ids":["semeval2015.d001.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"addition%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018889999999999998,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.459123083972372}
{"item_id":"semeval2015.d001.s012.t001","gold_sense_keys":["subtraction%1:04:01::"],"candidates":[{"index":1,"sense_key":"subtraction%1:04:01::","synset_id":"subtraction.n.01"},{"index":2,"sense_key":"subtraction%1:04:00::","synset_id":"subtraction.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"subtraction%1:04:01::","call_ids":["semeval2015.d001.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"subtraction%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":390,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002563,"input_uncached_usd":0.0000195,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.177958499989472}
{"item_id":"semeval2015.d001.s012.t002","gold_sense_keys":["multiplication%1:04:00::"],"candidates":[{"index":1,"sense_key":"multiplication%1:04:01::","synset_id":"generation.n.07"},{"index":2,"sense_key":"multiplication%1:22:00::","synset_id":"multiplication.n.02"},{"index":3,"sense_key":"multiplication%1:04:00::","synset_id":"multiplication.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"multiplication%1:04:00::","call_ids":["semeval2015.d001.s012.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"multiplication%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001556,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1389948329888284}
{"item_id":"semeval2015.d001.s012.t003","gold_sense_keys":["division%1:04:01::"],"candidates":[{"index":1,"sense_key":"division%1:14:00::","synset_id":"division.n.01"},{"index":2,"sense_key":"division%1:09:00::","synset_id":"part.n.09"},{"index":3,"sense_key":"division%1:04:00::","synset_id":"division.n.03"},{"index":4,"sense_key":"division%1:14:03::","synset_id":"division.n.04"},{"index":5,"sense_key":"division%1:10:00::","synset_id":"division.n.05"},{"index":6,"sense_key":"division%1:14:06::","synset_id":"class.n.05"},{"index":7,"sense_key":"division%1:14:05::","synset_id":"division.n.07"},{"index":8,"sense_key":"division%1:14:04::","synset_id":"division.n.08"},{"index":9,"sense_key":"division%1:14:02::","synset_id":"division.n.09"},{"index":10,"sense_key":"division%1:14:01::","synset_id":"division.n.10"},{"index":11,"sense_key":"division%1:04:01::","synset_id":"division.n.11"},{"index":12,"sense_key":"division%1:04:02::","synset_id":"division.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"division%1:04:01::","call_ids":["semeval2015.d001.s012.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"division%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027229999999999995,"input_uncached_usd":0.000035499999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.287082374969032}
{"item_id":"semeval2015.d001.s013.t000","gold_sense_keys":["power%1:10:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"power%1:10:00::","call_ids":["semeval2015.d001.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"power%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002182,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.038022540975362}
{"item_id":"semeval2015.d001.s013.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003158,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.301710832980461}
{"item_id":"semeval2015.d001.s014.t001","gold_sense_keys":["possible%3:00:00::"],"candidates":[{"index":1,"sense_key":"possible%3:00:00::","synset_id":"possible.a.01"},{"index":2,"sense_key":"possible%3:00:04::","synset_id":"potential.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"possible%3:00:00::","call_ids":["semeval2015.d001.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"possible%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":336,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004583999999999999,"input_uncached_usd":0.0000168,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.726902166963555}
{"item_id":"semeval2015.d001.s014.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s014.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049455,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.285770874994341}
{"item_id":"semeval2015.d001.s015.t000","gold_sense_keys":["power%1:10:00::"],"candidates":[{"index":1,"sense_key":"power%1:07:00::","synset_id":"power.n.01"},{"index":2,"sense_key":"power%1:19:00::","synset_id":"power.n.02"},{"index":3,"sense_key":"power%1:09:01::","synset_id":"ability.n.02"},{"index":4,"sense_key":"power%1:26:00::","synset_id":"office.n.04"},{"index":5,"sense_key":"power%1:18:00::","synset_id":"power.n.05"},{"index":6,"sense_key":"power%1:10:00::","synset_id":"exponent.n.03"},{"index":7,"sense_key":"power%1:07:02::","synset_id":"might.n.01"},{"index":8,"sense_key":"power%1:14:00::","synset_id":"world_power.n.01"},{"index":9,"sense_key":"power%1:18:01::","synset_id":"baron.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"power%1:10:00::","call_ids":["semeval2015.d001.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"power%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031915000000000003,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.988621000025887}
{"item_id":"semeval2015.d001.s015.t001","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034935,"input_uncached_usd":0.00003575,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.86787541594822}
{"item_id":"semeval2015.d001.s015.t003","gold_sense_keys":["root%1:23:00::"],"candidates":[{"index":1,"sense_key":"root%1:20:00::","synset_id":"root.n.01"},{"index":2,"sense_key":"root%1:15:00::","synset_id":"beginning.n.04"},{"index":3,"sense_key":"root%1:10:00::","synset_id":"root.n.03"},{"index":4,"sense_key":"root%1:23:00::","synset_id":"root.n.04"},{"index":5,"sense_key":"root%1:14:00::","synset_id":"solution.n.04"},{"index":6,"sense_key":"root%1:18:00::","synset_id":"ancestor.n.01"},{"index":7,"sense_key":"root%1:10:01::","synset_id":"etymon.n.01"},{"index":8,"sense_key":"root%1:08:00::","synset_id":"root.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"root%1:23:00::","call_ids":["semeval2015.d001.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"root%1:23:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":589,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021505,"input_uncached_usd":0.000029449999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5601089170086198}
{"item_id":"semeval2015.d001.s015.t004","gold_sense_keys":["too%4:02:01::"],"candidates":[{"index":1,"sense_key":"too%4:02:00::","synset_id":"excessively.r.01"},{"index":2,"sense_key":"too%4:02:01::","synset_id":"besides.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"too%4:02:01::","call_ids":["semeval2015.d001.s015.t004__v1__a1","semeval2015.d001.s015.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"too%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":695,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00058515,"input_uncached_usd":0.000034750000000000004,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.97014166705776}
{"item_id":"semeval2015.d001.s016.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2015.d001.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":707,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037454999999999997,"input_uncached_usd":0.00003535,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.331069541047327}
{"item_id":"semeval2015.d001.s016.t001","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":5,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"},{"index":6,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":7,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"specify%2:32:02::","call_ids":["semeval2015.d001.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"specify%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":579,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.00095695,"input_uncached_usd":0.00002895,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.897529417008627}
{"item_id":"semeval2015.d001.s016.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"},{"index":3,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":4,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":374,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012749999999999998,"input_uncached_usd":0.0000187,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.501324334007222}
{"item_id":"semeval2015.d001.s016.t004","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:10:00::","call_ids":["semeval2015.d001.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":2640,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.00108465,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.0010559999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.070053542032838}
{"item_id":"semeval2015.d001.s017.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s017.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.77527345798444}
{"item_id":"semeval2015.d001.s017.t001","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":5,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"},{"index":6,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":7,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"specify%2:32:02::","call_ids":["semeval2015.d001.s017.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"specify%2:32:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044504999999999995,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.943748958001379}
{"item_id":"semeval2015.d001.s017.t002","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"high%5:00:00:superior:01","call_ids":["semeval2015.d001.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"high%5:00:00:superior:01","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036499999999999993,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.851013957988471}
{"item_id":"semeval2015.d001.s017.t003","gold_sense_keys":["priority%1:26:00::"],"candidates":[{"index":1,"sense_key":"priority%1:26:00::","synset_id":"precedence.n.01"},{"index":2,"sense_key":"priority%1:07:00::","synset_id":"priority.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"priority%1:26:00::","call_ids":["semeval2015.d001.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"priority%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":327,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00027874999999999997,"input_uncached_usd":0.000016349999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.082793666981161}
{"item_id":"semeval2015.d001.s018.t001","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["semeval2015.d001.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1044,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004426,"input_uncached_usd":0.000052199999999999995,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.259947500017006}
{"item_id":"semeval2015.d001.s018.t002","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:10:00::","call_ids":["semeval2015.d001.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001894,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5795674999826588}
{"item_id":"semeval2015.d001.s018.t003","gold_sense_keys":["check%2:32:10::"],"candidates":[{"index":1,"sense_key":"check%2:31:00::","synset_id":"check.v.01"},{"index":2,"sense_key":"check%2:41:04::","synset_id":"check.v.02"},{"index":3,"sense_key":"check%2:31:02::","synset_id":"see.v.10"},{"index":4,"sense_key":"check%2:41:08::","synset_id":"control.v.02"},{"index":5,"sense_key":"check%2:38:15::","synset_id":"check.v.05"},{"index":6,"sense_key":"check%2:31:01::","synset_id":"check.v.06"},{"index":7,"sense_key":"check%2:30:02::","synset_id":"check.v.07"},{"index":8,"sense_key":"check%2:42:09::","synset_id":"check.v.08"},{"index":9,"sense_key":"check%2:42:08::","synset_id":"match.v.01"},{"index":10,"sense_key":"check%2:41:14::","synset_id":"check.v.10"},{"index":11,"sense_key":"check%2:41:02::","synset_id":"discipline.v.01"},{"index":12,"sense_key":"check%2:40:13::","synset_id":"check.v.12"},{"index":13,"sense_key":"check%2:40:00::","synset_id":"check.v.13"},{"index":14,"sense_key":"check%2:38:14::","synset_id":"check.v.14"},{"index":15,"sense_key":"check%2:38:12::","synset_id":"check.v.15"},{"index":16,"sense_key":"check%2:36:00::","synset_id":"check.v.16"},{"index":17,"sense_key":"check%2:33:02::","synset_id":"check.v.17"},{"index":18,"sense_key":"check%2:33:06::","synset_id":"check.v.18"},{"index":19,"sense_key":"check%2:33:00::","synset_id":"check.v.19"},{"index":20,"sense_key":"check%2:32:07::","synset_id":"check.v.20"},{"index":21,"sense_key":"check%2:32:10::","synset_id":"determine.v.08"},{"index":22,"sense_key":"check%2:31:12::","synset_id":"check.v.22"},{"index":23,"sense_key":"check%2:30:10::","synset_id":"check.v.23"},{"index":24,"sense_key":"check%2:30:09::","synset_id":"check.v.24"},{"index":25,"sense_key":"check%2:30:03::","synset_id":"crack.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"check%2:31:00::","call_ids":["semeval2015.d001.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"check%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1332,"cached_input_tokens":0,"output_tokens":2064,"reasoning_output_tokens":2048},"cost":{"currency":"USD","total_usd":0.0008922,"input_uncached_usd":0.00006659999999999999,"input_cached_usd":0.0,"output_usd":0.0008256,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.691363124991767}
{"item_id":"semeval2015.d001.s018.t004","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":420,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023219999999999998,"input_uncached_usd":0.000021,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.337314500007778}
{"item_id":"semeval2015.d001.s019.t000","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"treat%2:31:00::","call_ids":["semeval2015.d001.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034464999999999995,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.159354000003077}
{"item_id":"semeval2015.d001.s019.t001","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":419,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020654999999999998,"input_uncached_usd":0.000020949999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.904005291988142}
{"item_id":"semeval2015.d001.s019.t003","gold_sense_keys":["treat%2:31:00::"],"candidates":[{"index":1,"sense_key":"treat%2:41:00::","synset_id":"treat.v.01"},{"index":2,"sense_key":"treat%2:30:01::","synset_id":"process.v.01"},{"index":3,"sense_key":"treat%2:29:00::","synset_id":"treat.v.03"},{"index":4,"sense_key":"treat%2:32:00::","synset_id":"cover.v.05"},{"index":5,"sense_key":"treat%2:40:00::","synset_id":"treat.v.05"},{"index":6,"sense_key":"treat%2:34:00::","synset_id":"regale.v.01"},{"index":7,"sense_key":"treat%2:32:03::","synset_id":"treat.v.07"},{"index":8,"sense_key":"treat%2:31:00::","synset_id":"treat.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"treat%2:31:00::","call_ids":["semeval2015.d001.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"treat%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003702499999999999,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.445148082973901}
{"item_id":"semeval2015.d001.s019.t004","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d001.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":343,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035634999999999996,"input_uncached_usd":0.00001715,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.9184278750326484}
{"item_id":"semeval2015.d001.s019.t005","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"function%1:10:00::","call_ids":["semeval2015.d001.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"function%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":588,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047099999999999996,"input_uncached_usd":0.0000294,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.009371458028909}
{"item_id":"semeval2015.d001.s020.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00062155,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.145891625026707}
{"item_id":"semeval2015.d001.s020.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s020.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036104999999999997,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.204392999992706}
{"item_id":"semeval2015.d001.s020.t002","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049205,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.111350457998924}
{"item_id":"semeval2015.d001.s022.t000","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"way%1:07:01::","call_ids":["semeval2015.d001.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"way%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":740,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045299999999999995,"input_uncached_usd":0.000037,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.059295458020642}
{"item_id":"semeval2015.d001.s022.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":461,"cached_input_tokens":0,"output_tokens":1938,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00079825,"input_uncached_usd":0.000023049999999999998,"input_cached_usd":0.0,"output_usd":0.0007752,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.036421208002139}
{"item_id":"semeval2015.d001.s022.t002","gold_sense_keys":["conditional%3:00:00::"],"candidates":[{"index":1,"sense_key":"conditional%5:00:00:qualified:02","synset_id":"conditional.s.01"},{"index":2,"sense_key":"conditional%3:00:00::","synset_id":"conditional.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"conditional%3:00:00::","call_ids":["semeval2015.d001.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"conditional%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":330,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045809999999999997,"input_uncached_usd":0.000016499999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.0831965419929475}
{"item_id":"semeval2015.d001.s022.t003","gold_sense_keys":["operation%1:04:01::"],"candidates":[{"index":1,"sense_key":"operation%1:26:00::","synset_id":"operation.n.01"},{"index":2,"sense_key":"operation%1:04:06::","synset_id":"operation.n.02"},{"index":3,"sense_key":"operation%1:04:03::","synset_id":"operation.n.03"},{"index":4,"sense_key":"operation%1:22:00::","synset_id":"operation.n.04"},{"index":5,"sense_key":"operation%1:04:02::","synset_id":"operation.n.05"},{"index":6,"sense_key":"operation%1:04:00::","synset_id":"operation.n.06"},{"index":7,"sense_key":"operation%1:04:05::","synset_id":"operation.n.07"},{"index":8,"sense_key":"operation%1:22:01::","synset_id":"operation.n.08"},{"index":9,"sense_key":"operation%1:09:00::","synset_id":"process.n.02"},{"index":10,"sense_key":"operation%1:04:01::","synset_id":"mathematical_process.n.01"},{"index":11,"sense_key":"operation%1:04:07::","synset_id":"operation.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"operation%1:04:05::","call_ids":["semeval2015.d001.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"operation%1:04:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":829,"cached_input_tokens":0,"output_tokens":2577,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.0010722499999999999,"input_uncached_usd":0.00004145,"input_cached_usd":0.0,"output_usd":0.0010308,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":16.374289291037712}
{"item_id":"semeval2015.d001.s023.t000","gold_sense_keys":["introduce%2:30:03::","introduce%2:35:00::"],"candidates":[{"index":1,"sense_key":"introduce%2:32:00::","synset_id":"introduce.v.01"},{"index":2,"sense_key":"introduce%2:36:00::","synset_id":"introduce.v.02"},{"index":3,"sense_key":"introduce%2:30:03::","synset_id":"insert.v.02"},{"index":4,"sense_key":"introduce%2:38:00::","synset_id":"bring_in.v.01"},{"index":5,"sense_key":"introduce%2:36:01::","synset_id":"introduce.v.05"},{"index":6,"sense_key":"introduce%2:35:00::","synset_id":"insert.v.01"},{"index":7,"sense_key":"introduce%2:32:02::","synset_id":"introduce.v.07"},{"index":8,"sense_key":"introduce%2:31:00::","synset_id":"introduce.v.08"},{"index":9,"sense_key":"introduce%2:32:01::","synset_id":"precede.v.05"},{"index":10,"sense_key":"introduce%2:30:00::","synset_id":"inaugurate.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"introduce%2:35:00::","call_ids":["semeval2015.d001.s023.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"introduce%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":1745,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007333,"input_uncached_usd":0.0000353,"input_cached_usd":0.0,"output_usd":0.0006979999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.354640542005654}
{"item_id":"semeval2015.d001.s023.t002","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041775,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.46019716700539}
{"item_id":"semeval2015.d001.s023.t005","gold_sense_keys":["true%3:00:00::"],"candidates":[{"index":1,"sense_key":"true%3:00:00::","synset_id":"true.a.01"},{"index":2,"sense_key":"true%5:00:00:accurate:00","synset_id":"true.s.01"},{"index":3,"sense_key":"true%5:00:02:faithful:00","synset_id":"true.s.02"},{"index":4,"sense_key":"true%3:00:04::","synset_id":"truthful.a.01"},{"index":5,"sense_key":"true%5:00:00:typical:00","synset_id":"true.s.03"},{"index":6,"sense_key":"true%5:00:00:trustworthy:00","synset_id":"dependable.s.01"},{"index":7,"sense_key":"true%5:00:00:sincere:00","synset_id":"genuine.s.01"},{"index":8,"sense_key":"true%5:00:00:real:02","synset_id":"true.s.06"},{"index":9,"sense_key":"true%5:00:00:geographic:00","synset_id":"true.s.07"},{"index":10,"sense_key":"true%5:00:00:legitimate:00","synset_id":"true.s.08"},{"index":11,"sense_key":"true%5:00:00:harmonious:00","synset_id":"on-key.s.01"},{"index":12,"sense_key":"true%5:00:00:even:01","synset_id":"true.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"true%3:00:00::","call_ids":["semeval2015.d001.s023.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"true%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":783,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006091499999999999,"input_uncached_usd":0.000039149999999999996,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.6503679159795865}
{"item_id":"semeval2015.d001.s023.t006","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"find%2:39:02::","call_ids":["semeval2015.d001.s023.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"find%2:39:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1058,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006481000000000001,"input_uncached_usd":0.0000529,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.394564541988075}
{"item_id":"semeval2015.d001.s023.t010","gold_sense_keys":["instance%1:11:00::"],"candidates":[{"index":1,"sense_key":"instance%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"instance%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"instance%1:11:00::","call_ids":["semeval2015.d001.s023.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"instance%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":363,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00053655,"input_uncached_usd":0.00001815,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.868540666007902}
{"item_id":"semeval2015.d001.s025.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s025.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036524999999999996,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0179044579854235}
{"item_id":"semeval2015.d001.s025.t001","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s025.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":431,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036114999999999997,"input_uncached_usd":0.00002155,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.857917916961014}
{"item_id":"semeval2015.d001.s026.t001","gold_sense_keys":["useful%3:00:00::"],"candidates":[{"index":1,"sense_key":"useful%3:00:00::","synset_id":"useful.a.01"},{"index":2,"sense_key":"useful%5:00:00:functional:00","synset_id":"utilitarian.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"useful%5:00:00:functional:00","call_ids":["semeval2015.d001.s026.t001__v1__a1","semeval2015.d001.s026.t001__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"useful%5:00:00:functional:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":1762,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007392499999999999,"input_uncached_usd":0.00003445,"input_cached_usd":0.0,"output_usd":0.0007048,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.296826625009999}
{"item_id":"semeval2015.d001.s026.t002","gold_sense_keys":["work%2:41:02::"],"candidates":[{"index":1,"sense_key":"work%2:41:02::","synset_id":"work.v.01"},{"index":2,"sense_key":"work%2:41:00::","synset_id":"work.v.02"},{"index":3,"sense_key":"work%2:41:03::","synset_id":"work.v.03"},{"index":4,"sense_key":"work%2:35:02::","synset_id":"function.v.01"},{"index":5,"sense_key":"work%2:36:00::","synset_id":"work.v.05"},{"index":6,"sense_key":"work%2:29:00::","synset_id":"exercise.v.03"},{"index":7,"sense_key":"work%2:38:00::","synset_id":"make.v.36"},{"index":8,"sense_key":"work%2:41:04::","synset_id":"work.v.08"},{"index":9,"sense_key":"work%2:41:05::","synset_id":"work.v.09"},{"index":10,"sense_key":"work%2:38:02::","synset_id":"work.v.10"},{"index":11,"sense_key":"work%2:36:01::","synset_id":"bring.v.03"},{"index":12,"sense_key":"work%2:41:01::","synset_id":"work.v.12"},{"index":13,"sense_key":"work%2:36:09::","synset_id":"cultivate.v.02"},{"index":14,"sense_key":"work%2:42:10::","synset_id":"work.v.14"},{"index":15,"sense_key":"work%2:41:10::","synset_id":"influence.v.01"},{"index":16,"sense_key":"work%2:41:11::","synset_id":"work.v.16"},{"index":17,"sense_key":"work%2:41:12::","synset_id":"work.v.17"},{"index":18,"sense_key":"work%2:39:10::","synset_id":"work.v.18"},{"index":19,"sense_key":"work%2:37:10::","synset_id":"work.v.19"},{"index":20,"sense_key":"work%2:36:12::","synset_id":"shape.v.02"},{"index":21,"sense_key":"work%2:35:14::","synset_id":"work.v.21"},{"index":22,"sense_key":"work%2:35:00::","synset_id":"knead.v.01"},{"index":23,"sense_key":"work%2:34:10::","synset_id":"exploit.v.01"},{"index":24,"sense_key":"work%2:31:13::","synset_id":"solve.v.01"},{"index":25,"sense_key":"work%2:30:14::","synset_id":"ferment.v.03"},{"index":26,"sense_key":"work%2:30:13::","synset_id":"sour.v.01"},{"index":27,"sense_key":"work%2:30:10::","synset_id":"work.v.27"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"work%2:35:02::","call_ids":["semeval2015.d001.s026.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"work%2:35:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1417,"cached_input_tokens":0,"output_tokens":2001,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00087125,"input_uncached_usd":0.00007085,"input_cached_usd":0.0,"output_usd":0.0008003999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.84397737501422}
{"item_id":"semeval2015.d001.s027.t000","gold_sense_keys":["now%4:02:07::"],"candidates":[{"index":1,"sense_key":"now%4:02:05::","synset_id":"now.r.01"},{"index":2,"sense_key":"now%4:02:00::","synset_id":"nowadays.r.01"},{"index":3,"sense_key":"now%4:02:02::","synset_id":"now.r.03"},{"index":4,"sense_key":"now%4:02:04::","synset_id":"now.r.04"},{"index":5,"sense_key":"now%4:02:01::","synset_id":"immediately.r.01"},{"index":6,"sense_key":"now%4:02:07::","synset_id":"now.r.06"},{"index":7,"sense_key":"now%4:02:06::","synset_id":"now.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"now%4:02:07::","call_ids":["semeval2015.d001.s027.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"now%4:02:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031675000000000003,"input_uncached_usd":0.000028749999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.355551625019871}
{"item_id":"semeval2015.d001.s027.t001","gold_sense_keys":["ask%2:32:04::"],"candidates":[{"index":1,"sense_key":"ask%2:32:00::","synset_id":"ask.v.01"},{"index":2,"sense_key":"ask%2:32:01::","synset_id":"ask.v.02"},{"index":3,"sense_key":"ask%2:32:02::","synset_id":"ask.v.03"},{"index":4,"sense_key":"ask%2:32:05::","synset_id":"ask.v.04"},{"index":5,"sense_key":"ask%2:32:04::","synset_id":"ask.v.05"},{"index":6,"sense_key":"ask%2:42:00::","synset_id":"necessitate.v.01"},{"index":7,"sense_key":"ask%2:32:09::","synset_id":"ask.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ask%2:32:02::","call_ids":["semeval2015.d001.s027.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ask%2:32:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":578,"cached_input_tokens":0,"output_tokens":3088,"reasoning_output_tokens":3072},"cost":{"currency":"USD","total_usd":0.0012640999999999998,"input_uncached_usd":0.000028899999999999998,"input_cached_usd":0.0,"output_usd":0.0012351999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.554683499969542}
{"item_id":"semeval2015.d001.s027.t002","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d001.s027.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":359,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020355,"input_uncached_usd":0.00001795,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.027909042022657}
{"item_id":"semeval2015.d001.s028.t000","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":2,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":3,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":4,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":5,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":6,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":7,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":8,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":9,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":10,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":11,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":12,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"easy%3:00:02::","call_ids":["semeval2015.d001.s028.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"easy%3:00:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":776,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004548,"input_uncached_usd":0.0000388,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.865597291965969}
{"item_id":"semeval2015.d001.s028.t002","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s028.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":598,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031790000000000003,"input_uncached_usd":0.0000299,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.77652733301511}
{"item_id":"semeval2015.d001.s028.t003","gold_sense_keys":["cos%1:24:00::"],"candidates":[{"index":1,"sense_key":"cos%1:24:00::","synset_id":"cosine.n.01"},{"index":2,"sense_key":"cos%1:13:00::","synset_id":"cos.n.02"},{"index":3,"sense_key":"carbon_monoxide%1:27:00::","synset_id":"carbon_monoxide.n.01"},{"index":4,"sense_key":"cobalt%1:27:00::","synset_id":"cobalt.n.01"},{"index":5,"sense_key":"conscientious_objector%1:18:00::","synset_id":"conscientious_objector.n.01"},{"index":6,"sense_key":"colorado%1:15:00::","synset_id":"colorado.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"carbon_monoxide%1:27:00::","call_ids":["semeval2015.d001.s028.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"carbon_monoxide%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002625,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.522200834006071}
{"item_id":"semeval2015.d001.s028.t004","gold_sense_keys":["sin%1:24:00::"],"candidates":[{"index":1,"sense_key":"sin%1:07:00::","synset_id":"sin.n.01"},{"index":2,"sense_key":"sin%1:04:00::","synset_id":"sin.n.02"},{"index":3,"sense_key":"sin%1:24:00::","synset_id":"sine.n.01"},{"index":4,"sense_key":"sin%1:18:00::","synset_id":"sin.n.04"},{"index":5,"sense_key":"sin%1:10:00::","synset_id":"sin.n.05"},{"index":6,"sense_key":"sin%1:04:01::","synset_id":"sin.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"sin%1:24:00::","call_ids":["semeval2015.d001.s028.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"sin%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.0000808,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.6859400420216843}
{"item_id":"semeval2015.d001.s028.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d001.s028.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028409999999999997,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.196811292029452}
{"item_id":"semeval2015.d001.s028.t008","gold_sense_keys":["sum%1:09:01::"],"candidates":[{"index":1,"sense_key":"sum%1:21:00::","synset_id":"sum.n.01"},{"index":2,"sense_key":"sum%1:09:01::","synset_id":"sum.n.02"},{"index":3,"sense_key":"sum%1:14:00::","synset_id":"sum.n.03"},{"index":4,"sense_key":"sum%1:09:00::","synset_id":"kernel.n.03"},{"index":5,"sense_key":"sum%1:06:00::","synset_id":"sum.n.05"},{"index":6,"sense_key":"sum%1:14:01::","synset_id":"union.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sum%1:09:01::","call_ids":["semeval2015.d001.s028.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sum%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006982,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.056231583992485}
{"item_id":"semeval2015.d001.s028.t009","gold_sense_keys":["product%1:09:00::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"product%1:09:00::","call_ids":["semeval2015.d001.s028.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"product%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002126,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5734277089941315}
{"item_id":"semeval2015.d001.s030.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s030.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036539999999999994,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.7851889170124196}
{"item_id":"semeval2015.d001.s030.t001","gold_sense_keys":["plus%1:04:00::"],"candidates":[{"index":1,"sense_key":"plus%1:07:00::","synset_id":"asset.n.01"},{"index":2,"sense_key":"plus%1:04:00::","synset_id":"summation.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"plus%1:04:00::","call_ids":["semeval2015.d001.s030.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"plus%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":323,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030415,"input_uncached_usd":0.00001615,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9917031250079162}
{"item_id":"semeval2015.d001.s030.t002","gold_sense_keys":["times%1:04:00::"],"candidates":[{"index":1,"sense_key":"times%1:28:01::","synset_id":"times.n.01"},{"index":2,"sense_key":"times%1:04:00::","synset_id":"multiplication.n.03"},{"index":3,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":4,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":5,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":6,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":7,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":8,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":9,"sense_key":"clock_time%1:28:00::","synset_id":"clock_time.n.01"},{"index":10,"sense_key":"fourth_dimension%1:28:00::","synset_id":"fourth_dimension.n.01"},{"index":11,"sense_key":"meter%1:07:00::","synset_id":"meter.n.04"},{"index":12,"sense_key":"prison_term%1:28:00::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"times%1:04:00::","call_ids":["semeval2015.d001.s030.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"times%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":773,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024985,"input_uncached_usd":0.00003865,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.361540417012293}
{"item_id":"semeval2015.d001.s030.t003","gold_sense_keys":["have%2:40:00::","have%2:42:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["semeval2015.d001.s030.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1078,"cached_input_tokens":0,"output_tokens":1297,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005726999999999999,"input_uncached_usd":0.000053899999999999996,"input_cached_usd":0.0,"output_usd":0.0005187999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.488850667024963}
{"item_id":"semeval2015.d001.s030.t004","gold_sense_keys":["operator%1:24:00::"],"candidates":[{"index":1,"sense_key":"operator%1:24:00::","synset_id":"operator.n.01"},{"index":2,"sense_key":"operator%1:18:00::","synset_id":"operator.n.02"},{"index":3,"sense_key":"operator%1:18:01::","synset_id":"operator.n.03"},{"index":4,"sense_key":"operator%1:18:02::","synset_id":"hustler.n.02"},{"index":5,"sense_key":"operator%1:18:03::","synset_id":"operator.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"operator%1:24:00::","call_ids":["semeval2015.d001.s030.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"operator%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":411,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015495,"input_uncached_usd":0.000020549999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.751448791997973}
{"item_id":"semeval2015.d001.s031.t000","gold_sense_keys":["useful%3:00:00::"],"candidates":[{"index":1,"sense_key":"useful%3:00:00::","synset_id":"useful.a.01"},{"index":2,"sense_key":"useful%5:00:00:functional:00","synset_id":"utilitarian.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"useful%5:00:00:functional:00","call_ids":["semeval2015.d001.s031.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"useful%5:00:00:functional:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":323,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045774999999999993,"input_uncached_usd":0.00001615,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.298077249957714}
{"item_id":"semeval2015.d001.s032.t000","gold_sense_keys":["there%4:02:00::"],"candidates":[{"index":1,"sense_key":"there%4:02:00::","synset_id":"there.r.01"},{"index":2,"sense_key":"there%4:02:02::","synset_id":"there.r.02"},{"index":3,"sense_key":"there%4:02:01::","synset_id":"there.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"there%4:02:00::","call_ids":["semeval2015.d001.s032.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"there%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00066535,"input_uncached_usd":0.00001895,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.2991432499839}
{"item_id":"semeval2015.d001.s032.t001","gold_sense_keys":["have%2:42:12::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"have%2:40:00::","call_ids":["semeval2015.d001.s032.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"have%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1096,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005732000000000001,"input_uncached_usd":0.0000548,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.633321666973643}
{"item_id":"semeval2015.d001.s032.t002","gold_sense_keys":["list%1:10:00::"],"candidates":[{"index":1,"sense_key":"list%1:10:00::","synset_id":"list.n.01"},{"index":2,"sense_key":"list%1:07:00::","synset_id":"tilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"list%1:10:00::","call_ids":["semeval2015.d001.s032.t002__v1__a1","semeval2015.d001.s032.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"list%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":749,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048545,"input_uncached_usd":0.00003745,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.029746958985925}
{"item_id":"semeval2015.d001.s032.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"},{"index":3,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":4,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s032.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":397,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041025,"input_uncached_usd":0.000019849999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.271224791998975}
{"item_id":"semeval2015.d001.s033.t000","gold_sense_keys":["see%2:39:00::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"see%2:39:03::","call_ids":["semeval2015.d001.s033.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"see%2:39:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1410,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00048649999999999995,"input_uncached_usd":0.00007049999999999999,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.72347779199481}
{"item_id":"semeval2015.d001.s033.t001","gold_sense_keys":["let%2:41:00::"],"candidates":[{"index":1,"sense_key":"let%2:41:00::","synset_id":"let.v.01"},{"index":2,"sense_key":"let%2:32:00::","synset_id":"let.v.02"},{"index":3,"sense_key":"let%2:32:02::","synset_id":"permit.v.01"},{"index":4,"sense_key":"let%2:30:01::","synset_id":"get.v.03"},{"index":5,"sense_key":"let%2:30:00::","synset_id":"let.v.05"},{"index":6,"sense_key":"let%2:40:00::","synset_id":"lease.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"let%2:41:00::","call_ids":["semeval2015.d001.s033.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"let%2:41:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00075015,"input_uncached_usd":0.000026949999999999998,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.897475834004581}
{"item_id":"semeval2015.d001.s033.t002","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":3,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":4,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":5,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":6,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":7,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":8,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":9,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":10,"sense_key":"change%2:30:06::","synset_id":"change.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%2:30:01::","call_ids":["semeval2015.d001.s033.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":765,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00053105,"input_uncached_usd":0.00003825,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.380992749996949}
{"item_id":"semeval2015.d001.s033.t003","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s033.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056995,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.229338375036605}
{"item_id":"semeval2015.d001.s033.t004","gold_sense_keys":["just%4:02:00::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"just%4:02:00::","call_ids":["semeval2015.d001.s033.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"just%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":554,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003669,"input_uncached_usd":0.0000277,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.694048417033628}
{"item_id":"semeval2015.d001.s033.t005","gold_sense_keys":["way%1:04:01::","way%1:07:01::"],"candidates":[{"index":1,"sense_key":"way%1:07:01::","synset_id":"manner.n.01"},{"index":2,"sense_key":"way%1:04:01::","synset_id":"means.n.01"},{"index":3,"sense_key":"way%1:15:00::","synset_id":"direction.n.01"},{"index":4,"sense_key":"way%1:26:00::","synset_id":"way.n.04"},{"index":5,"sense_key":"way%1:04:00::","synset_id":"way.n.05"},{"index":6,"sense_key":"way%1:06:00::","synset_id":"way.n.06"},{"index":7,"sense_key":"way%1:04:02::","synset_id":"way.n.07"},{"index":8,"sense_key":"way%1:23:00::","synset_id":"room.n.02"},{"index":9,"sense_key":"way%1:07:02::","synset_id":"way.n.09"},{"index":10,"sense_key":"way%1:09:00::","synset_id":"way.n.10"},{"index":11,"sense_key":"way%1:09:01::","synset_id":"way.n.11"},{"index":12,"sense_key":"way%1:21:00::","synset_id":"way.n.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"way%1:04:01::","call_ids":["semeval2015.d001.s033.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"way%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":737,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029925,"input_uncached_usd":0.00003685,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.362874374957755}
{"item_id":"semeval2015.d001.s033.t007","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"log%1:10:01::","call_ids":["semeval2015.d001.s033.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"log%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":442,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004125,"input_uncached_usd":0.0000221,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.988348417042289}
{"item_id":"semeval2015.d001.s034.t000","gold_sense_keys":["time%1:11:00::"],"candidates":[{"index":1,"sense_key":"time%1:11:00::","synset_id":"time.n.01"},{"index":2,"sense_key":"time%1:28:05::","synset_id":"time.n.02"},{"index":3,"sense_key":"time%1:28:00::","synset_id":"time.n.03"},{"index":4,"sense_key":"time%1:28:06::","synset_id":"time.n.04"},{"index":5,"sense_key":"time%1:03:00::","synset_id":"time.n.05"},{"index":6,"sense_key":"time%1:11:01::","synset_id":"time.n.06"},{"index":7,"sense_key":"time%1:28:03::","synset_id":"clock_time.n.01"},{"index":8,"sense_key":"time%1:28:04::","synset_id":"fourth_dimension.n.01"},{"index":9,"sense_key":"time%1:07:01::","synset_id":"meter.n.04"},{"index":10,"sense_key":"time%1:28:01::","synset_id":"prison_term.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"time%1:11:00::","call_ids":["semeval2015.d001.s034.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"time%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039715,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.788405125029385}
{"item_id":"semeval2015.d001.s034.t002","gold_sense_keys":["expression%1:10:03::"],"candidates":[{"index":1,"sense_key":"expression%1:07:00::","synset_id":"expression.n.01"},{"index":2,"sense_key":"expression%1:10:02::","synset_id":"expression.n.02"},{"index":3,"sense_key":"expression%1:10:04::","synset_id":"expression.n.03"},{"index":4,"sense_key":"expression%1:10:00::","synset_id":"saying.n.01"},{"index":5,"sense_key":"expression%1:10:01::","synset_id":"formulation.n.03"},{"index":6,"sense_key":"expression%1:10:03::","synset_id":"formula.n.01"},{"index":7,"sense_key":"expression%1:22:00::","synset_id":"expression.n.07"},{"index":8,"sense_key":"expression%1:10:05::","synset_id":"construction.n.02"},{"index":9,"sense_key":"expression%1:04:00::","synset_id":"expression.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"expression%1:10:03::","call_ids":["semeval2015.d001.s034.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"expression%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":626,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026809999999999996,"input_uncached_usd":0.0000313,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9744504579575732}
{"item_id":"semeval2015.d001.s034.t003","gold_sense_keys":["variable%1:10:00::"],"candidates":[{"index":1,"sense_key":"variable%1:17:00::","synset_id":"variable.n.01"},{"index":2,"sense_key":"variable%1:09:00::","synset_id":"variable.n.02"},{"index":3,"sense_key":"variable%1:17:01::","synset_id":"variable_star.n.01"},{"index":4,"sense_key":"variable%1:10:00::","synset_id":"variable.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"variable%1:10:00::","call_ids":["semeval2015.d001.s034.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"variable%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004094,"input_uncached_usd":0.000018999999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.009046541992575}
{"item_id":"semeval2015.d001.s034.t004","gold_sense_keys":["value%1:09:00::"],"candidates":[{"index":1,"sense_key":"value%1:09:00::","synset_id":"value.n.01"},{"index":2,"sense_key":"value%1:07:00::","synset_id":"value.n.02"},{"index":3,"sense_key":"value%1:21:00::","synset_id":"value.n.03"},{"index":4,"sense_key":"value%1:07:01::","synset_id":"value.n.04"},{"index":5,"sense_key":"value%1:28:00::","synset_id":"value.n.05"},{"index":6,"sense_key":"value%1:09:01::","synset_id":"value.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"value%1:09:00::","call_ids":["semeval2015.d001.s034.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"value%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":495,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015915,"input_uncached_usd":0.00002475,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.563469041022472}
{"item_id":"semeval2015.d001.s034.t005","gold_sense_keys":["change%2:30:01::"],"candidates":[{"index":1,"sense_key":"change%2:30:01::","synset_id":"change.v.01"},{"index":2,"sense_key":"change%2:30:00::","synset_id":"change.v.02"},{"index":3,"sense_key":"change%2:30:02::","synset_id":"change.v.03"},{"index":4,"sense_key":"change%2:30:05::","synset_id":"switch.v.03"},{"index":5,"sense_key":"change%2:30:03::","synset_id":"change.v.05"},{"index":6,"sense_key":"change%2:30:04::","synset_id":"change.v.06"},{"index":7,"sense_key":"change%2:40:00::","synset_id":"exchange.v.01"},{"index":8,"sense_key":"change%2:38:00::","synset_id":"transfer.v.06"},{"index":9,"sense_key":"change%2:30:08::","synset_id":"deepen.v.04"},{"index":10,"sense_key":"change%2:30:06::","synset_id":"change.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"change%2:30:01::","call_ids":["semeval2015.d001.s034.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"change%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":741,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055545,"input_uncached_usd":0.00003705,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.8284862919827}
{"item_id":"semeval2015.d001.s034.t006","gold_sense_keys":["last%3:00:00::","last%5:00:00:past:00"],"candidates":[{"index":1,"sense_key":"last%5:00:00:past:00","synset_id":"last.s.01"},{"index":2,"sense_key":"last%3:00:00::","synset_id":"last.a.02"},{"index":3,"sense_key":"last%5:00:00:closing:00","synset_id":"concluding.s.01"},{"index":4,"sense_key":"last%5:00:00:unlikely:00","synset_id":"last.s.03"},{"index":5,"sense_key":"last%5:00:00:dying:00","synset_id":"last.s.04"},{"index":6,"sense_key":"last%5:00:00:ultimate:00","synset_id":"final.s.02"},{"index":7,"sense_key":"last%5:00:00:high:02","synset_id":"last.s.06"},{"index":8,"sense_key":"last%5:00:00:unalterable:00","synset_id":"final.s.03"},{"index":9,"sense_key":"last%5:00:00:worst:00","synset_id":"last.s.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"last%3:00:00::","call_ids":["semeval2015.d001.s034.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"last%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":655,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00055115,"input_uncached_usd":0.000032749999999999996,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.138430457969662}
{"item_id":"semeval2015.d001.s034.t007","gold_sense_keys":["result%1:10:00::","result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"result%1:10:00::","call_ids":["semeval2015.d001.s034.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"result%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00051535,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.707101874984801}
{"item_id":"semeval2015.d001.s035.t000","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s035.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":721,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00070805,"input_uncached_usd":0.000036049999999999995,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.029805792029947}
{"item_id":"semeval2015.d001.s035.t001","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s035.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":566,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029069999999999996,"input_uncached_usd":0.0000283,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.279612916987389}
{"item_id":"semeval2015.d001.s036.t000","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s036.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025699999999999996,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.478917499945965}
{"item_id":"semeval2015.d001.s037.t000","gold_sense_keys":["load%2:40:12::"],"candidates":[{"index":1,"sense_key":"load%2:35:01::","synset_id":"load.v.01"},{"index":2,"sense_key":"load%2:35:02::","synset_id":"load.v.02"},{"index":3,"sense_key":"load%2:40:12::","synset_id":"load.v.03"},{"index":4,"sense_key":"load%2:35:10::","synset_id":"load.v.04"},{"index":5,"sense_key":"load%2:30:12::","synset_id":"load.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"load%2:40:12::","call_ids":["semeval2015.d001.s037.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"load%2:40:12::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":437,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023344999999999998,"input_uncached_usd":0.00002185,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.079461208020803}
{"item_id":"semeval2015.d001.s038.t000","gold_sense_keys":["execute%2:41:04::"],"candidates":[{"index":1,"sense_key":"execute%2:41:00::","synset_id":"execute.v.01"},{"index":2,"sense_key":"execute%2:41:01::","synset_id":"execute.v.02"},{"index":3,"sense_key":"execute%2:36:00::","synset_id":"carry_through.v.01"},{"index":4,"sense_key":"execute%2:41:03::","synset_id":"execute.v.04"},{"index":5,"sense_key":"execute%2:41:04::","synset_id":"run.v.19"},{"index":6,"sense_key":"execute%2:36:01::","synset_id":"perform.v.01"},{"index":7,"sense_key":"execute%2:32:00::","synset_id":"execute.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"execute%2:41:04::","call_ids":["semeval2015.d001.s038.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"execute%2:41:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004167,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.835736333974637}
{"item_id":"semeval2015.d001.s038.t001","gold_sense_keys":["instruction%1:10:02::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"instruction%1:10:02::","call_ids":["semeval2015.d001.s038.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"instruction%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020664999999999998,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.68558008299442}
{"item_id":"semeval2015.d001.s038.t002","gold_sense_keys":["file%1:10:00::"],"candidates":[{"index":1,"sense_key":"file%1:10:00::","synset_id":"file.n.01"},{"index":2,"sense_key":"file%1:14:00::","synset_id":"file.n.02"},{"index":3,"sense_key":"file%1:06:01::","synset_id":"file.n.03"},{"index":4,"sense_key":"file%1:06:00::","synset_id":"file.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"file%1:10:00::","call_ids":["semeval2015.d001.s038.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"file%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":367,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028115,"input_uncached_usd":0.00001835,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.435048374987673}
{"item_id":"semeval2015.d001.s039.t000","gold_sense_keys":["nice%3:00:00::"],"candidates":[{"index":1,"sense_key":"nice%3:00:00::","synset_id":"nice.a.01"},{"index":2,"sense_key":"nice%5:00:00:respectable:00","synset_id":"decent.s.01"},{"index":3,"sense_key":"nice%5:00:02:precise:00","synset_id":"nice.s.02"},{"index":4,"sense_key":"nice%5:00:00:fastidious:00","synset_id":"dainty.s.04"},{"index":5,"sense_key":"nice%5:00:00:polite:00","synset_id":"courteous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"nice%3:00:00::","call_ids":["semeval2015.d001.s039.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"nice%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.00072105,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.569962833018508}
{"item_id":"semeval2015.d001.s039.t001","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s039.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00074355,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.661130249965936}
{"item_id":"semeval2015.d001.s039.t002","gold_sense_keys":["define%2:32:01::"],"candidates":[{"index":1,"sense_key":"define%2:42:00::","synset_id":"specify.v.03"},{"index":2,"sense_key":"define%2:32:00::","synset_id":"define.v.02"},{"index":3,"sense_key":"define%2:42:01::","synset_id":"define.v.03"},{"index":4,"sense_key":"define%2:42:03::","synset_id":"define.v.04"},{"index":5,"sense_key":"define%2:32:01::","synset_id":"specify.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"define%2:32:01::","call_ids":["semeval2015.d001.s039.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"define%2:32:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003849,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.055546957999468}
{"item_id":"semeval2015.d001.s039.t003","gold_sense_keys":["library%1:14:01::"],"candidates":[{"index":1,"sense_key":"library%1:06:01::","synset_id":"library.n.01"},{"index":2,"sense_key":"library%1:14:00::","synset_id":"library.n.02"},{"index":3,"sense_key":"library%1:06:00::","synset_id":"library.n.03"},{"index":4,"sense_key":"library%1:14:01::","synset_id":"library.n.04"},{"index":5,"sense_key":"library%1:06:02::","synset_id":"library.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"library%1:14:01::","call_ids":["semeval2015.d001.s039.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"library%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":379,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00035814999999999995,"input_uncached_usd":0.00001895,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.779101749998517}
{"item_id":"semeval2015.d001.s039.t004","gold_sense_keys":["resume%2:30:00::"],"candidates":[{"index":1,"sense_key":"resume%2:30:00::","synset_id":"resume.v.01"},{"index":2,"sense_key":"resume%2:30:01::","synset_id":"resume.v.02"},{"index":3,"sense_key":"resume%2:41:00::","synset_id":"resume.v.03"},{"index":4,"sense_key":"resume%2:32:00::","synset_id":"sum_up.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"resume%2:30:00::","call_ids":["semeval2015.d001.s039.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"resume%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":360,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002292,"input_uncached_usd":0.000018,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.861435375001747}
{"item_id":"semeval2015.d001.s039.t005","gold_sense_keys":["previous%5:00:00:preceding:00"],"candidates":[{"index":1,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"},{"index":2,"sense_key":"previous%5:00:00:past:00","synset_id":"former.s.02"},{"index":3,"sense_key":"previous%5:00:00:early:00","synset_id":"previous.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"previous%5:00:00:preceding:00","call_ids":["semeval2015.d001.s039.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"previous%5:00:00:preceding:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":350,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020309999999999998,"input_uncached_usd":0.0000175,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.272915417037439}
{"item_id":"semeval2015.d001.s039.t006","gold_sense_keys":["work%1:04:00::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2015.d001.s039.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":570,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004189,"input_uncached_usd":0.000028499999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.060898875002749}
{"item_id":"semeval2015.d001.s040.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s040.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031925,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.474662207998335}
{"item_id":"semeval2015.d001.s041.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"save%2:40:00::","call_ids":["semeval2015.d001.s041.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"save%2:40:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":639,"cached_input_tokens":0,"output_tokens":2704,"reasoning_output_tokens":2688},"cost":{"currency":"USD","total_usd":0.0011135499999999998,"input_uncached_usd":0.00003195,"input_cached_usd":0.0,"output_usd":0.0010816,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.827495082980022}
{"item_id":"semeval2015.d001.s041.t001","gold_sense_keys":["instruction%1:10:02::"],"candidates":[{"index":1,"sense_key":"instruction%1:10:04::","synset_id":"direction.n.06"},{"index":2,"sense_key":"instruction%1:04:01::","synset_id":"education.n.01"},{"index":3,"sense_key":"instruction%1:04:00::","synset_id":"teaching.n.01"},{"index":4,"sense_key":"instruction%1:10:02::","synset_id":"instruction.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"instruction%1:10:02::","call_ids":["semeval2015.d001.s041.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"instruction%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030845,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.430414332950022}
{"item_id":"semeval2015.d001.s041.t002","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s041.t002__v1__a1","semeval2015.d001.s041.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":864,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037745,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.00034559999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.317660250002518}
{"item_id":"semeval2015.d001.s041.t003","gold_sense_keys":["begin%2:42:00::","begin%2:42:04::"],"candidates":[{"index":1,"sense_key":"begin%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"begin%2:42:00::","synset_id":"begin.v.02"},{"index":3,"sense_key":"begin%2:30:01::","synset_id":"begin.v.03"},{"index":4,"sense_key":"begin%2:32:03::","synset_id":"begin.v.04"},{"index":5,"sense_key":"begin%2:42:02::","synset_id":"begin.v.05"},{"index":6,"sense_key":"begin%2:42:04::","synset_id":"begin.v.06"},{"index":7,"sense_key":"begin%2:42:03::","synset_id":"begin.v.07"},{"index":8,"sense_key":"begin%2:41:04::","synset_id":"begin.v.08"},{"index":9,"sense_key":"begin%2:41:00::","synset_id":"begin.v.09"},{"index":10,"sense_key":"begin%2:32:04::","synset_id":"begin.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"begin%2:42:04::","call_ids":["semeval2015.d001.s041.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"begin%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":727,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.00052915,"input_uncached_usd":0.000036349999999999996,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.578761750017293}
{"item_id":"semeval2015.d001.s041.t004","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d001.s041.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":405,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028264999999999996,"input_uncached_usd":0.000020249999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.670752749952953}
{"item_id":"semeval2015.d001.s042.t000","gold_sense_keys":["generate%2:36:00::"],"candidates":[{"index":1,"sense_key":"generate%2:36:00::","synset_id":"generate.v.01"},{"index":2,"sense_key":"generate%2:36:02::","synset_id":"render.v.04"},{"index":3,"sense_key":"generate%2:36:01::","synset_id":"generate.v.03"},{"index":4,"sense_key":"generate%2:29:00::","synset_id":"beget.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"generate%2:36:00::","call_ids":["semeval2015.d001.s042.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"generate%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005903500000000001,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.4696430409676395}
{"item_id":"semeval2015.d001.s042.t002","gold_sense_keys":["easy%3:00:01::"],"candidates":[{"index":1,"sense_key":"easy%3:00:01::","synset_id":"easy.a.01"},{"index":2,"sense_key":"easy%5:00:00:unhurried:00","synset_id":"easy.s.01"},{"index":3,"sense_key":"easy%3:00:02::","synset_id":"easy.a.03"},{"index":4,"sense_key":"easy%5:00:00:pleasing:00","synset_id":"easy.s.02"},{"index":5,"sense_key":"easy%5:00:00:light:08","synset_id":"easy.s.03"},{"index":6,"sense_key":"easy%5:00:00:impressionable:00","synset_id":"easy.s.04"},{"index":7,"sense_key":"easy%5:00:00:rich:00","synset_id":"comfortable.s.03"},{"index":8,"sense_key":"easy%5:00:00:gradual:02","synset_id":"easy.s.06"},{"index":9,"sense_key":"easy%5:00:00:comfortable:00","synset_id":"easy.s.07"},{"index":10,"sense_key":"easy%5:00:00:unchaste:00","synset_id":"easy.s.08"},{"index":11,"sense_key":"easy%5:00:00:available:00","synset_id":"easy.s.09"},{"index":12,"sense_key":"easy%5:00:00:abundant:00","synset_id":"easy.s.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"easy%3:00:01::","call_ids":["semeval2015.d001.s042.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"easy%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":735,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017114999999999999,"input_uncached_usd":0.00003675,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.854801207955461}
{"item_id":"semeval2015.d001.s042.t003","gold_sense_keys":["fix%2:30:01::"],"candidates":[{"index":1,"sense_key":"fix%2:30:01::","synset_id":"repair.v.01"},{"index":2,"sense_key":"fix%2:35:00::","synset_id":"fasten.v.01"},{"index":3,"sense_key":"fix%2:32:03::","synset_id":"specify.v.02"},{"index":4,"sense_key":"fix%2:36:00::","synset_id":"cook.v.02"},{"index":5,"sense_key":"fix%2:33:00::","synset_id":"pay_back.v.02"},{"index":6,"sense_key":"fix%2:36:01::","synset_id":"fix.v.06"},{"index":7,"sense_key":"fix%2:30:02::","synset_id":"fix.v.07"},{"index":8,"sense_key":"fix%2:30:00::","synset_id":"fixate.v.03"},{"index":9,"sense_key":"fix%2:29:00::","synset_id":"sterilize.v.02"},{"index":10,"sense_key":"fix%2:41:04::","synset_id":"fix.v.10"},{"index":11,"sense_key":"fix%2:35:01::","synset_id":"situate.v.02"},{"index":12,"sense_key":"fix%2:30:03::","synset_id":"fix.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"fix%2:30:01::","call_ids":["semeval2015.d001.s042.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"fix%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":1425,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006089,"input_uncached_usd":0.0000389,"input_cached_usd":0.0,"output_usd":0.00057,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.284641583974008}
{"item_id":"semeval2015.d001.s042.t004","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s042.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003902,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.5358814999926835}
{"item_id":"semeval2015.d001.s042.t005","gold_sense_keys":["text_editor%1:10:00::"],"candidates":[{"index":1,"sense_key":"text_editor%1:18:00::","synset_id":"copy_editor.n.01"},{"index":2,"sense_key":"text_editor%1:10:00::","synset_id":"text_editor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"text_editor%1:10:00::","call_ids":["semeval2015.d001.s042.t005__v1__a1","semeval2015.d001.s042.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"text_editor%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":617,"cached_input_tokens":0,"output_tokens":865,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037684999999999997,"input_uncached_usd":0.00003085,"input_cached_usd":0.0,"output_usd":0.00034599999999999995,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.967663333984092}
{"item_id":"semeval2015.d001.s043.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s043.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":653,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034625,"input_uncached_usd":0.00003265,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.897483584005386}
{"item_id":"semeval2015.d001.s043.t001","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s043.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005900500000000001,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.94375245901756}
{"item_id":"semeval2015.d001.s044.t000","gold_sense_keys":["save%2:32:04::"],"candidates":[{"index":1,"sense_key":"save%2:41:01::","synset_id":"salvage.v.01"},{"index":2,"sense_key":"save%2:40:00::","synset_id":"save.v.02"},{"index":3,"sense_key":"save%2:41:00::","synset_id":"save.v.03"},{"index":4,"sense_key":"save%2:40:02::","synset_id":"save.v.04"},{"index":5,"sense_key":"save%2:40:01::","synset_id":"save.v.05"},{"index":6,"sense_key":"save%2:30:00::","synset_id":"save.v.06"},{"index":7,"sense_key":"save%2:41:02::","synset_id":"deliver.v.08"},{"index":8,"sense_key":"save%2:41:05::","synset_id":"spare.v.01"},{"index":9,"sense_key":"save%2:40:04::","synset_id":"save.v.09"},{"index":10,"sense_key":"save%2:40:03::","synset_id":"keep_open.v.01"},{"index":11,"sense_key":"save%2:32:04::","synset_id":"write.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":11,"chosen_sense_key":"save%2:32:04::","call_ids":["semeval2015.d001.s044.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":11,"predicted_sense_key":"save%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008067500000000001,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.83566995797446}
{"item_id":"semeval2015.d001.s044.t001","gold_sense_keys":["log%1:10:02::"],"candidates":[{"index":1,"sense_key":"log%1:27:01::","synset_id":"log.n.01"},{"index":2,"sense_key":"log%1:10:01::","synset_id":"logarithm.n.01"},{"index":3,"sense_key":"log%1:10:02::","synset_id":"log.n.03"},{"index":4,"sense_key":"log%1:10:00::","synset_id":"log.n.04"},{"index":5,"sense_key":"log%1:06:01::","synset_id":"log.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"log%1:10:02::","call_ids":["semeval2015.d001.s044.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"log%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033374999999999995,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.328979709011037}
{"item_id":"semeval2015.d001.s044.t002","gold_sense_keys":["result%1:10:00::","result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"result%1:11:00::","call_ids":["semeval2015.d001.s044.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"result%1:11:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":436,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023299999999999997,"input_uncached_usd":0.000021799999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.172740041045472}
{"item_id":"semeval2015.d001.s044.t004","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"able%3:00:00::","call_ids":["semeval2015.d001.s044.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"able%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023184999999999997,"input_uncached_usd":0.00002065,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.92129179101903}
{"item_id":"semeval2015.d001.s044.t005","gold_sense_keys":["print%2:36:00::"],"candidates":[{"index":1,"sense_key":"print%2:36:02::","synset_id":"print.v.01"},{"index":2,"sense_key":"print%2:36:03::","synset_id":"print.v.02"},{"index":3,"sense_key":"print%2:36:01::","synset_id":"print.v.03"},{"index":4,"sense_key":"print%2:36:00::","synset_id":"print.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"print%2:36:02::","call_ids":["semeval2015.d001.s044.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"print%2:36:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045904999999999996,"input_uncached_usd":0.00001745,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.299527292023413}
{"item_id":"semeval2015.d001.s044.t006","gold_sense_keys":["publish%2:32:00::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d001.s044.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005112500000000001,"input_uncached_usd":0.000018449999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.557849917036947}
{"item_id":"semeval2015.d001.s045.t000","gold_sense_keys":["quit%2:42:04::"],"candidates":[{"index":1,"sense_key":"quit%2:42:04::","synset_id":"discontinue.v.01"},{"index":2,"sense_key":"quit%2:41:00::","synset_id":"leave_office.v.01"},{"index":3,"sense_key":"quit%2:38:00::","synset_id":"depart.v.04"},{"index":4,"sense_key":"quit%2:40:00::","synset_id":"foreswear.v.02"},{"index":5,"sense_key":"quit%2:33:05::","synset_id":"drop_out.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"quit%2:42:04::","call_ids":["semeval2015.d001.s045.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"quit%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":464,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007464,"input_uncached_usd":0.000023199999999999998,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.58673949999502}
{"item_id":"semeval2015.d001.s046.t000","gold_sense_keys":["program%1:10:02::"],"candidates":[{"index":1,"sense_key":"program%1:09:00::","synset_id":"plan.n.01"},{"index":2,"sense_key":"program%1:09:01::","synset_id":"program.n.02"},{"index":3,"sense_key":"program%1:10:01::","synset_id":"broadcast.n.02"},{"index":4,"sense_key":"program%1:10:04::","synset_id":"platform.n.02"},{"index":5,"sense_key":"program%1:10:00::","synset_id":"program.n.05"},{"index":6,"sense_key":"program%1:10:05::","synset_id":"course_of_study.n.01"},{"index":7,"sense_key":"program%1:10:02::","synset_id":"program.n.07"},{"index":8,"sense_key":"program%1:04:00::","synset_id":"program.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"program%1:10:02::","call_ids":["semeval2015.d001.s046.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"program%1:10:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":645,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019265,"input_uncached_usd":0.00003225,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.386508458002936}
{"item_id":"semeval2015.d001.s047.t000","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":913,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039185,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.0003652,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.701039124978706}
{"item_id":"semeval2015.d001.s047.t001","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"new%3:00:00::","call_ids":["semeval2015.d001.s047.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"new%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00045329999999999996,"input_uncached_usd":0.000036899999999999996,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.726412750023883}
{"item_id":"semeval2015.d001.s047.t003","gold_sense_keys":["do%2:36:01::","do%2:41:04::"],"candidates":[{"index":1,"sense_key":"do%2:41:01::","synset_id":"make.v.01"},{"index":2,"sense_key":"do%2:36:01::","synset_id":"perform.v.01"},{"index":3,"sense_key":"do%2:41:04::","synset_id":"do.v.03"},{"index":4,"sense_key":"do%2:42:00::","synset_id":"do.v.04"},{"index":5,"sense_key":"do%2:36:02::","synset_id":"cause.v.01"},{"index":6,"sense_key":"do%2:41:02::","synset_id":"practice.v.01"},{"index":7,"sense_key":"do%2:42:01::","synset_id":"suffice.v.01"},{"index":8,"sense_key":"do%2:36:00::","synset_id":"do.v.08"},{"index":9,"sense_key":"do%2:29:09::","synset_id":"act.v.02"},{"index":10,"sense_key":"do%2:42:02::","synset_id":"serve.v.09"},{"index":11,"sense_key":"do%2:41:03::","synset_id":"do.v.11"},{"index":12,"sense_key":"do%2:29:04::","synset_id":"dress.v.16"},{"index":13,"sense_key":"do%2:38:00::","synset_id":"do.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"do%2:36:01::","call_ids":["semeval2015.d001.s047.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"do%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":851,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00025374999999999996,"input_uncached_usd":0.00004255,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3589922499959357}
{"item_id":"semeval2015.d001.s047.t004","gold_sense_keys":["go%2:38:00::"],"candidates":[{"index":1,"sense_key":"go%2:38:00::","synset_id":"travel.v.01"},{"index":2,"sense_key":"go%2:41:00::","synset_id":"go.v.02"},{"index":3,"sense_key":"go%2:38:01::","synset_id":"go.v.03"},{"index":4,"sense_key":"go%2:30:04::","synset_id":"become.v.01"},{"index":5,"sense_key":"go%2:42:06::","synset_id":"go.v.05"},{"index":6,"sense_key":"go%2:42:02::","synset_id":"run.v.05"},{"index":7,"sense_key":"go%2:42:00::","synset_id":"run.v.03"},{"index":8,"sense_key":"go%2:30:01::","synset_id":"proceed.v.04"},{"index":9,"sense_key":"go%2:30:08::","synset_id":"go.v.09"},{"index":10,"sense_key":"go%2:42:07::","synset_id":"go.v.10"},{"index":11,"sense_key":"go%2:39:00::","synset_id":"sound.v.02"},{"index":12,"sense_key":"go%2:35:00::","synset_id":"function.v.01"},{"index":13,"sense_key":"go%2:34:00::","synset_id":"run_low.v.01"},{"index":14,"sense_key":"go%2:30:02::","synset_id":"move.v.13"},{"index":15,"sense_key":"go%2:42:14::","synset_id":"survive.v.01"},{"index":16,"sense_key":"go%2:42:03::","synset_id":"go.v.16"},{"index":17,"sense_key":"go%2:30:00::","synset_id":"die.v.01"},{"index":18,"sense_key":"go%2:42:04::","synset_id":"belong.v.03"},{"index":19,"sense_key":"go%2:42:10::","synset_id":"go.v.19"},{"index":20,"sense_key":"go%2:38:09::","synset_id":"start.v.09"},{"index":21,"sense_key":"go%2:33:00::","synset_id":"move.v.15"},{"index":22,"sense_key":"go%2:42:13::","synset_id":"go.v.22"},{"index":23,"sense_key":"go%2:42:12::","synset_id":"go.v.23"},{"index":24,"sense_key":"go%2:42:11::","synset_id":"blend.v.02"},{"index":25,"sense_key":"go%2:42:08::","synset_id":"go.v.25"},{"index":26,"sense_key":"go%2:42:05::","synset_id":"fit.v.02"},{"index":27,"sense_key":"go%2:35:13::","synset_id":"rifle.v.02"},{"index":28,"sense_key":"go%2:34:09::","synset_id":"go.v.28"},{"index":29,"sense_key":"go%2:31:12::","synset_id":"plump.v.04"},{"index":30,"sense_key":"go%2:30:03::","synset_id":"fail.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"go%2:38:00::","call_ids":["semeval2015.d001.s047.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"go%2:38:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1652,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0006265999999999999,"input_uncached_usd":0.0000826,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.84894329198869}
{"item_id":"semeval2015.d001.s047.t006","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":532,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.000417,"input_uncached_usd":0.0000266,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.942311124992557}
{"item_id":"semeval2015.d001.s047.t007","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s047.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049385,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.2149650839855894}
{"item_id":"semeval2015.d001.s047.t008","gold_sense_keys":["new%5:00:00:other:00"],"candidates":[{"index":1,"sense_key":"new%3:00:00::","synset_id":"new.a.01"},{"index":2,"sense_key":"new%5:00:00:original:00","synset_id":"fresh.s.03"},{"index":3,"sense_key":"new%5:00:00:inexperienced:00","synset_id":"raw.s.11"},{"index":4,"sense_key":"new%5:00:00:unprecedented:00","synset_id":"new.s.03"},{"index":5,"sense_key":"new%5:00:00:other:00","synset_id":"new.s.04"},{"index":6,"sense_key":"new%3:00:09::","synset_id":"new.a.06"},{"index":7,"sense_key":"new%5:00:02:original:00","synset_id":"newfangled.s.01"},{"index":8,"sense_key":"new%5:00:02:late:01","synset_id":"new.s.06"},{"index":9,"sense_key":"new%5:00:01:late:01","synset_id":"modern.s.04"},{"index":10,"sense_key":"new%5:00:00:early:02","synset_id":"new.s.08"},{"index":11,"sense_key":"new%5:00:00:unaccustomed:00","synset_id":"new.s.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"new%5:00:00:other:00","call_ids":["semeval2015.d001.s047.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"new%5:00:00:other:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":738,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005809,"input_uncached_usd":0.000036899999999999996,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.184608624957036}
{"item_id":"semeval2015.d001.s047.t009","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s047.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039434999999999996,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.375798292050604}
{"item_id":"semeval2015.d001.s048.t000","gold_sense_keys":["then%4:02:00::"],"candidates":[{"index":1,"sense_key":"then%4:02:00::","synset_id":"then.r.01"},{"index":2,"sense_key":"then%4:02:02::","synset_id":"then.r.02"},{"index":3,"sense_key":"then%4:02:01::","synset_id":"then.r.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"then%4:02:00::","call_ids":["semeval2015.d001.s048.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"then%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017915,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7951531250146218}
{"item_id":"semeval2015.d001.s048.t001","gold_sense_keys":["focus%1:09:00::"],"candidates":[{"index":1,"sense_key":"focus%1:09:00::","synset_id":"focus.n.01"},{"index":2,"sense_key":"focus%1:07:01::","synset_id":"focus.n.02"},{"index":3,"sense_key":"focus%1:07:03::","synset_id":"focus.n.03"},{"index":4,"sense_key":"focus%1:26:00::","synset_id":"focus.n.04"},{"index":5,"sense_key":"focus%1:26:01::","synset_id":"stress.n.03"},{"index":6,"sense_key":"focus%1:19:00::","synset_id":"focus.n.06"},{"index":7,"sense_key":"focus%1:15:00::","synset_id":"focus.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"focus%1:09:00::","call_ids":["semeval2015.d001.s048.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"focus%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":540,"cached_input_tokens":0,"output_tokens":721,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003154,"input_uncached_usd":0.000027,"input_cached_usd":0.0,"output_usd":0.00028839999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.279192749992944}
{"item_id":"semeval2015.d001.s048.t003","gold_sense_keys":["text%1:10:00::"],"candidates":[{"index":1,"sense_key":"text%1:10:00::","synset_id":"text.n.01"},{"index":2,"sense_key":"text%1:10:03::","synset_id":"text.n.02"},{"index":3,"sense_key":"text%1:10:02::","synset_id":"textbook.n.01"},{"index":4,"sense_key":"text%1:10:01::","synset_id":"text.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"text%1:10:00::","call_ids":["semeval2015.d001.s048.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"text%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036174999999999993,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.831003165978473}
{"item_id":"semeval2015.d001.s048.t005","gold_sense_keys":["type%2:32:00::"],"candidates":[{"index":1,"sense_key":"type%2:32:00::","synset_id":"type.v.01"},{"index":2,"sense_key":"type%2:31:00::","synset_id":"type.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"type%2:32:00::","call_ids":["semeval2015.d001.s048.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"type%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":331,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043254999999999997,"input_uncached_usd":0.00001655,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.76177912496496}
{"item_id":"semeval2015.d001.s048.t006","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s048.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":591,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044594999999999997,"input_uncached_usd":0.00002955,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.742127375036944}
{"item_id":"semeval2015.d001.s049.t000","gold_sense_keys":["want%2:37:00::"],"candidates":[{"index":1,"sense_key":"want%2:37:00::","synset_id":"desire.v.01"},{"index":2,"sense_key":"want%2:34:00::","synset_id":"want.v.02"},{"index":3,"sense_key":"want%2:35:00::","synset_id":"want.v.03"},{"index":4,"sense_key":"want%2:31:01::","synset_id":"want.v.04"},{"index":5,"sense_key":"want%2:42:01::","synset_id":"want.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"want%2:37:00::","call_ids":["semeval2015.d001.s049.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"want%2:37:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023459999999999998,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.551716375048272}
{"item_id":"semeval2015.d001.s049.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s049.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":553,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018764999999999998,"input_uncached_usd":0.000027649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.796660041960422}
{"item_id":"semeval2015.d001.s049.t002","gold_sense_keys":["typical%3:00:00::","typical%5:00:00:normal:01"],"candidates":[{"index":1,"sense_key":"typical%3:00:00::","synset_id":"typical.a.01"},{"index":2,"sense_key":"typical%5:00:00:characteristic:00","synset_id":"distinctive.s.01"},{"index":3,"sense_key":"typical%5:00:00:normal:01","synset_id":"typical.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"typical%5:00:00:normal:01","call_ids":["semeval2015.d001.s049.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"typical%5:00:00:normal:01","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012959999999999998,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.279964542016387}
{"item_id":"semeval2015.d001.s049.t004","gold_sense_keys":["necessary%3:00:00::"],"candidates":[{"index":1,"sense_key":"necessary%3:00:00::","synset_id":"necessary.a.01"},{"index":2,"sense_key":"necessary%5:00:00:inevitable:00","synset_id":"necessary.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"necessary%3:00:00::","call_ids":["semeval2015.d001.s049.t004__v1__a1","semeval2015.d001.s049.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"necessary%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":705,"cached_input_tokens":0,"output_tokens":3489,"reasoning_output_tokens":3456},"cost":{"currency":"USD","total_usd":0.00143085,"input_uncached_usd":0.00003525,"input_cached_usd":0.0,"output_usd":0.0013955999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":20.65723562502535}
{"item_id":"semeval2015.d001.s049.t005","gold_sense_keys":["specify%2:32:02::"],"candidates":[{"index":1,"sense_key":"specify%2:32:03::","synset_id":"stipulate.v.01"},{"index":2,"sense_key":"specify%2:32:02::","synset_id":"specify.v.02"},{"index":3,"sense_key":"specify%2:42:00::","synset_id":"specify.v.03"},{"index":4,"sense_key":"specify%2:32:00::","synset_id":"specify.v.04"},{"index":5,"sense_key":"specify%2:31:02::","synset_id":"pin_down.v.01"},{"index":6,"sense_key":"specify%2:31:00::","synset_id":"intend.v.02"},{"index":7,"sense_key":"specify%2:31:05::","synset_id":"assign.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"specify%2:42:00::","call_ids":["semeval2015.d001.s049.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"specify%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":614,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008051000000000001,"input_uncached_usd":0.0000307,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.508268208010122}
{"item_id":"semeval2015.d001.s050.t001","gold_sense_keys":["function%1:24:00::"],"candidates":[{"index":1,"sense_key":"function%1:24:00::","synset_id":"function.n.01"},{"index":2,"sense_key":"function%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"function%1:04:00::","synset_id":"function.n.03"},{"index":4,"sense_key":"function%1:24:01::","synset_id":"function.n.04"},{"index":5,"sense_key":"function%1:14:00::","synset_id":"function.n.05"},{"index":6,"sense_key":"function%1:11:00::","synset_id":"affair.n.03"},{"index":7,"sense_key":"function%1:10:00::","synset_id":"routine.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"function%1:24:00::","call_ids":["semeval2015.d001.s050.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"function%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":600,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007019999999999999,"input_uncached_usd":0.000029999999999999997,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.32732754200697}
{"item_id":"semeval2015.d001.s050.t003","gold_sense_keys":["display%2:39:00::"],"candidates":[{"index":1,"sense_key":"display%2:39:00::","synset_id":"expose.v.03"},{"index":2,"sense_key":"display%2:41:00::","synset_id":"display.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"display%2:39:00::","call_ids":["semeval2015.d001.s050.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"display%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":349,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020304999999999998,"input_uncached_usd":0.00001745,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.026540833001491}
{"item_id":"semeval2015.d001.s050.t005","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d001.s050.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":401,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001806,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.0001604,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.7146652080118656}
{"item_id":"semeval2015.d001.s050.t006","gold_sense_keys":["window%1:06:03::"],"candidates":[{"index":1,"sense_key":"window%1:06:00::","synset_id":"window.n.01"},{"index":2,"sense_key":"window%1:06:01::","synset_id":"window.n.02"},{"index":3,"sense_key":"window%1:06:02::","synset_id":"window.n.03"},{"index":4,"sense_key":"window%1:17:00::","synset_id":"window.n.04"},{"index":5,"sense_key":"window%1:28:00::","synset_id":"window.n.05"},{"index":6,"sense_key":"window%1:06:04::","synset_id":"windowpane.n.01"},{"index":7,"sense_key":"window%1:06:05::","synset_id":"window.n.07"},{"index":8,"sense_key":"window%1:06:03::","synset_id":"window.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"window%1:06:03::","call_ids":["semeval2015.d001.s050.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"window%1:06:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003943,"input_uncached_usd":0.0000295,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.732165125024039}
{"item_id":"semeval2015.d001.s051.t000","gold_sense_keys":["set%2:35:00::"],"candidates":[{"index":1,"sense_key":"set%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"set%2:31:04::","synset_id":"determine.v.03"},{"index":3,"sense_key":"set%2:32:00::","synset_id":"specify.v.02"},{"index":4,"sense_key":"set%2:32:01::","synset_id":"set.v.04"},{"index":5,"sense_key":"set%2:30:01::","synset_id":"set.v.05"},{"index":6,"sense_key":"set%2:35:02::","synset_id":"set.v.06"},{"index":7,"sense_key":"set%2:30:09::","synset_id":"fix.v.12"},{"index":8,"sense_key":"set%2:30:02::","synset_id":"set.v.08"},{"index":9,"sense_key":"set%2:36:01::","synset_id":"set.v.09"},{"index":10,"sense_key":"set%2:38:00::","synset_id":"set.v.10"},{"index":11,"sense_key":"set%2:36:00::","synset_id":"arrange.v.06"},{"index":12,"sense_key":"set%2:35:09::","synset_id":"plant.v.01"},{"index":13,"sense_key":"set%2:36:12::","synset_id":"set.v.13"},{"index":14,"sense_key":"set%2:30:07::","synset_id":"jell.v.01"},{"index":15,"sense_key":"set%2:36:11::","synset_id":"typeset.v.01"},{"index":16,"sense_key":"set%2:35:15::","synset_id":"set.v.16"},{"index":17,"sense_key":"set%2:35:05::","synset_id":"set.v.17"},{"index":18,"sense_key":"set%2:35:04::","synset_id":"set.v.18"},{"index":19,"sense_key":"set%2:33:00::","synset_id":"sic.v.01"},{"index":20,"sense_key":"set%2:31:13::","synset_id":"place.v.11"},{"index":21,"sense_key":"set%2:30:06::","synset_id":"rig.v.04"},{"index":22,"sense_key":"set%2:30:12::","synset_id":"set_up.v.04"},{"index":23,"sense_key":"set%2:30:00::","synset_id":"adjust.v.01"},{"index":24,"sense_key":"set%2:29:06::","synset_id":"fructify.v.03"},{"index":25,"sense_key":"set%2:29:05::","synset_id":"dress.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%2:35:00::","call_ids":["semeval2015.d001.s051.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1238,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00027309999999999997,"input_uncached_usd":0.0000619,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.223891250032466}
{"item_id":"semeval2015.d001.s051.t001","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["semeval2015.d001.s051.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":403,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012895,"input_uncached_usd":0.00002015,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.1069118329905905}
{"item_id":"semeval2015.d001.s051.t003","gold_sense_keys":["same%3:00:02::"],"candidates":[{"index":1,"sense_key":"same%3:00:02::","synset_id":"same.a.01"},{"index":2,"sense_key":"same%3:00:00::","synset_id":"same.a.02"},{"index":3,"sense_key":"same%3:00:04::","synset_id":"like.a.02"},{"index":4,"sense_key":"same%5:00:00:unchanged:00","synset_id":"same.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"same%3:00:02::","call_ids":["semeval2015.d001.s051.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"same%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046324999999999996,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.329859457968269}
{"item_id":"semeval2015.d001.s052.t000","gold_sense_keys":["just%4:02:00::"],"candidates":[{"index":1,"sense_key":"just%4:02:00::","synset_id":"merely.r.01"},{"index":2,"sense_key":"just%4:02:05::","synset_id":"precisely.r.01"},{"index":3,"sense_key":"just%4:02:01::","synset_id":"just.r.03"},{"index":4,"sense_key":"just%4:02:03::","synset_id":"just.r.04"},{"index":5,"sense_key":"just%4:02:04::","synset_id":"barely.r.01"},{"index":6,"sense_key":"just%4:02:06::","synset_id":"just.r.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"just%4:02:00::","call_ids":["semeval2015.d001.s052.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"just%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044415,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.764586708042771}
{"item_id":"semeval2015.d001.s052.t001","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d001.s052.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002902,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.914492166019045}
{"item_id":"semeval2015.d001.s052.t002","gold_sense_keys":["add%2:30:00::"],"candidates":[{"index":1,"sense_key":"add%2:30:00::","synset_id":"add.v.01"},{"index":2,"sense_key":"add%2:32:01::","synset_id":"add.v.02"},{"index":3,"sense_key":"add%2:40:00::","synset_id":"lend.v.01"},{"index":4,"sense_key":"add%2:31:00::","synset_id":"add.v.04"},{"index":5,"sense_key":"add%2:32:00::","synset_id":"total.v.02"},{"index":6,"sense_key":"add%2:42:00::","synset_id":"add.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"add%2:30:00::","call_ids":["semeval2015.d001.s052.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"add%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021319999999999998,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8705750000081025}
{"item_id":"semeval2015.d001.s052.t003","gold_sense_keys":["list%1:10:00::"],"candidates":[{"index":1,"sense_key":"list%1:10:00::","synset_id":"list.n.01"},{"index":2,"sense_key":"list%1:07:00::","synset_id":"tilt.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"list%1:10:00::","call_ids":["semeval2015.d001.s052.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"list%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":365,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00045984999999999993,"input_uncached_usd":0.00001825,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.906674332974944}
{"item_id":"semeval2015.d001.s052.t004","gold_sense_keys":["mode%1:26:00::"],"candidates":[{"index":1,"sense_key":"mode%1:07:00::","synset_id":"manner.n.01"},{"index":2,"sense_key":"mode%1:26:00::","synset_id":"mode.n.02"},{"index":3,"sense_key":"mode%1:24:00::","synset_id":"modality.n.01"},{"index":4,"sense_key":"mode%1:24:02::","synset_id":"mood.n.03"},{"index":5,"sense_key":"mode%1:10:00::","synset_id":"mode.n.05"},{"index":6,"sense_key":"mode%1:09:00::","synset_id":"mode.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mode%1:26:00::","call_ids":["semeval2015.d001.s052.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mode%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":497,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023605,"input_uncached_usd":0.000024849999999999998,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1733460829709657}
{"item_id":"semeval2015.d001.s053.t000","gold_sense_keys":["set%2:32:00::"],"candidates":[{"index":1,"sense_key":"set%2:35:00::","synset_id":"put.v.01"},{"index":2,"sense_key":"set%2:31:04::","synset_id":"determine.v.03"},{"index":3,"sense_key":"set%2:32:00::","synset_id":"specify.v.02"},{"index":4,"sense_key":"set%2:32:01::","synset_id":"set.v.04"},{"index":5,"sense_key":"set%2:30:01::","synset_id":"set.v.05"},{"index":6,"sense_key":"set%2:35:02::","synset_id":"set.v.06"},{"index":7,"sense_key":"set%2:30:09::","synset_id":"fix.v.12"},{"index":8,"sense_key":"set%2:30:02::","synset_id":"set.v.08"},{"index":9,"sense_key":"set%2:36:01::","synset_id":"set.v.09"},{"index":10,"sense_key":"set%2:38:00::","synset_id":"set.v.10"},{"index":11,"sense_key":"set%2:36:00::","synset_id":"arrange.v.06"},{"index":12,"sense_key":"set%2:35:09::","synset_id":"plant.v.01"},{"index":13,"sense_key":"set%2:36:12::","synset_id":"set.v.13"},{"index":14,"sense_key":"set%2:30:07::","synset_id":"jell.v.01"},{"index":15,"sense_key":"set%2:36:11::","synset_id":"typeset.v.01"},{"index":16,"sense_key":"set%2:35:15::","synset_id":"set.v.16"},{"index":17,"sense_key":"set%2:35:05::","synset_id":"set.v.17"},{"index":18,"sense_key":"set%2:35:04::","synset_id":"set.v.18"},{"index":19,"sense_key":"set%2:33:00::","synset_id":"sic.v.01"},{"index":20,"sense_key":"set%2:31:13::","synset_id":"place.v.11"},{"index":21,"sense_key":"set%2:30:06::","synset_id":"rig.v.04"},{"index":22,"sense_key":"set%2:30:12::","synset_id":"set_up.v.04"},{"index":23,"sense_key":"set%2:30:00::","synset_id":"adjust.v.01"},{"index":24,"sense_key":"set%2:29:06::","synset_id":"fructify.v.03"},{"index":25,"sense_key":"set%2:29:05::","synset_id":"dress.v.16"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"set%2:35:00::","call_ids":["semeval2015.d001.s053.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"set%2:35:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1202,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006297000000000001,"input_uncached_usd":0.0000601,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.735045958019327}
{"item_id":"semeval2015.d001.s053.t002","gold_sense_keys":["color%1:07:00::","color%1:07:01::"],"candidates":[{"index":1,"sense_key":"color%1:07:00::","synset_id":"color.n.01"},{"index":2,"sense_key":"color%1:07:03::","synset_id":"color.n.02"},{"index":3,"sense_key":"color%1:07:02::","synset_id":"color.n.03"},{"index":4,"sense_key":"color%1:14:00::","synset_id":"color.n.04"},{"index":5,"sense_key":"color%1:07:04::","synset_id":"semblance.n.01"},{"index":6,"sense_key":"color%1:27:00::","synset_id":"coloring_material.n.01"},{"index":7,"sense_key":"color%1:09:01::","synset_id":"color.n.07"},{"index":8,"sense_key":"color%1:07:01::","synset_id":"color.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"color%1:07:01::","call_ids":["semeval2015.d001.s053.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"color%1:07:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054965,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.311728166998364}
{"item_id":"semeval2015.d002.s000.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":527,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016074999999999998,"input_uncached_usd":0.00002635,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.540169749991037}
{"item_id":"semeval2015.d002.s000.t002","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d002.s000.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":469,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031145,"input_uncached_usd":0.000023449999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.927516708034091}
{"item_id":"semeval2015.d002.s000.t003","gold_sense_keys":["organise%2:36:00::","organise%2:41:00::"],"candidates":[{"index":1,"sense_key":"organise%2:30:00::","synset_id":"organize.v.04"},{"index":2,"sense_key":"organise%2:41:01::","synset_id":"form.v.01"},{"index":3,"sense_key":"organise%2:41:02::","synset_id":"unionize.v.02"},{"index":4,"sense_key":"organise%2:41:00::","synset_id":"organize.v.02"},{"index":5,"sense_key":"organise%2:36:00::","synset_id":"organize.v.05"},{"index":6,"sense_key":"organise%2:31:00::","synset_id":"mastermind.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"organise%2:36:00::","call_ids":["semeval2015.d002.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"organise%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":462,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005671,"input_uncached_usd":0.0000231,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.66276866599219}
{"item_id":"semeval2015.d002.s000.t004","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":304,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00037999999999999997,"input_uncached_usd":0.0000152,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.452324874990154}
{"item_id":"semeval2015.d002.s000.t005","gold_sense_keys":["deal%2:32:08::"],"candidates":[{"index":1,"sense_key":"deal%2:32:08::","synset_id":"cover.v.05"},{"index":2,"sense_key":"deal%2:31:10::","synset_id":"consider.v.03"},{"index":3,"sense_key":"deal%2:41:09::","synset_id":"deal.v.03"},{"index":4,"sense_key":"deal%2:41:00::","synset_id":"cope.v.01"},{"index":5,"sense_key":"deal%2:40:01::","synset_id":"distribute.v.01"},{"index":6,"sense_key":"deal%2:40:00::","synset_id":"deal.v.06"},{"index":7,"sense_key":"deal%2:41:13::","synset_id":"manage.v.02"},{"index":8,"sense_key":"deal%2:41:04::","synset_id":"deal.v.08"},{"index":9,"sense_key":"deal%2:40:02::","synset_id":"deal.v.09"},{"index":10,"sense_key":"deal%2:41:02::","synset_id":"conduct.v.01"},{"index":11,"sense_key":"deal%2:40:07::","synset_id":"share.v.04"},{"index":12,"sense_key":"deal%2:40:06::","synset_id":"deal.v.12"},{"index":13,"sense_key":"deal%2:40:11::","synset_id":"deal.v.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"deal%2:41:09::","call_ids":["semeval2015.d002.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"deal%2:41:09::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":780,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032700000000000003,"input_uncached_usd":0.000039,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.090062625007704}
{"item_id":"semeval2015.d002.s000.t006","gold_sense_keys":["critical%5:00:00:crucial:00"],"candidates":[{"index":1,"sense_key":"critical%3:00:01::","synset_id":"critical.a.01"},{"index":2,"sense_key":"critical%3:00:04::","synset_id":"critical.a.02"},{"index":3,"sense_key":"critical%3:00:02::","synset_id":"critical.a.03"},{"index":4,"sense_key":"critical%5:00:00:indispensable:00","synset_id":"critical.s.01"},{"index":5,"sense_key":"critical%5:00:00:crucial:00","synset_id":"critical.s.02"},{"index":6,"sense_key":"critical%3:00:03::","synset_id":"critical.a.06"},{"index":7,"sense_key":"critical%3:01:00::","synset_id":"critical.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"critical%5:00:00:crucial:00","call_ids":["semeval2015.d002.s000.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"critical%5:00:00:crucial:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004154,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.744031041976996}
{"item_id":"semeval2015.d002.s000.t007","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2015.d002.s000.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004338,"input_uncached_usd":0.0000178,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.238597499963362}
{"item_id":"semeval2015.d002.s001.t000","gold_sense_keys":["participant%1:18:00::"],"candidates":[{"index":1,"sense_key":"participant%1:18:00::","synset_id":"participant.n.01"},{"index":2,"sense_key":"participant%1:18:01::","synset_id":"player.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participant%1:18:00::","call_ids":["semeval2015.d002.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participant%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":284,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00014859999999999998,"input_uncached_usd":0.0000142,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.520760833984241}
{"item_id":"semeval2015.d002.s001.t001","gold_sense_keys":["main%5:00:00:important:00"],"candidates":[{"index":1,"sense_key":"main%5:00:00:important:00","synset_id":"chief.s.01"},{"index":2,"sense_key":"main%3:00:02::","synset_id":"independent.a.03"},{"index":3,"sense_key":"main%5:00:00:intense:00","synset_id":"main.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"main%5:00:00:important:00","call_ids":["semeval2015.d002.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"main%5:00:00:important:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020414999999999997,"input_uncached_usd":0.00001855,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3900450420333073}
{"item_id":"semeval2015.d002.s001.t003","gold_sense_keys":["attend%2:42:00::"],"candidates":[{"index":1,"sense_key":"attend%2:42:00::","synset_id":"attend.v.01"},{"index":2,"sense_key":"attend%2:41:12::","synset_id":"attend.v.02"},{"index":3,"sense_key":"attend%2:42:01::","synset_id":"attend.v.03"},{"index":4,"sense_key":"attend%2:41:00::","synset_id":"serve.v.10"},{"index":5,"sense_key":"attend%2:39:00::","synset_id":"attend.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attend%2:42:00::","call_ids":["semeval2015.d002.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attend%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005174,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.234775332966819}
{"item_id":"semeval2015.d002.s001.t004","gold_sense_keys":["debate%2:32:00::"],"candidates":[{"index":1,"sense_key":"debate%2:31:00::","synset_id":"debate.v.01"},{"index":2,"sense_key":"debate%2:32:02::","synset_id":"consider.v.05"},{"index":3,"sense_key":"debate%2:32:00::","synset_id":"debate.v.03"},{"index":4,"sense_key":"debate%2:32:01::","synset_id":"argue.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"debate%2:32:00::","call_ids":["semeval2015.d002.s001.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"debate%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012795,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.2335328750195913}
{"item_id":"semeval2015.d002.s001.t005","gold_sense_keys":["issue%1:09:01::"],"candidates":[{"index":1,"sense_key":"issue%1:09:01::","synset_id":"issue.n.01"},{"index":2,"sense_key":"issue%1:10:00::","synset_id":"issue.n.02"},{"index":3,"sense_key":"issue%1:09:00::","synset_id":"topic.n.02"},{"index":4,"sense_key":"issue%1:04:01::","synset_id":"issue.n.04"},{"index":5,"sense_key":"issue%1:21:01::","synset_id":"issue.n.05"},{"index":6,"sense_key":"issue%1:21:00::","synset_id":"return.n.06"},{"index":7,"sense_key":"issue%1:19:00::","synset_id":"consequence.n.01"},{"index":8,"sense_key":"issue%1:18:00::","synset_id":"offspring.n.01"},{"index":9,"sense_key":"issue%1:11:00::","synset_id":"emergence.n.02"},{"index":10,"sense_key":"issue%1:06:00::","synset_id":"exit.n.01"},{"index":11,"sense_key":"issue%1:04:00::","synset_id":"issue.n.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"issue%1:09:00::","call_ids":["semeval2015.d002.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"issue%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":772,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019859999999999998,"input_uncached_usd":0.000038599999999999996,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1970164999947883}
{"item_id":"semeval2015.d002.s001.t008","gold_sense_keys":["actor%1:18:01::"],"candidates":[{"index":1,"sense_key":"actor%1:18:00::","synset_id":"actor.n.01"},{"index":2,"sense_key":"actor%1:18:01::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"actor%1:18:01::","call_ids":["semeval2015.d002.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"actor%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":314,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003805,"input_uncached_usd":0.0000157,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.176846458052751}
{"item_id":"semeval2015.d002.s001.t011","gold_sense_keys":["authority%1:18:01::"],"candidates":[{"index":1,"sense_key":"authority%1:07:00::","synset_id":"authority.n.01"},{"index":2,"sense_key":"authority%1:18:01::","synset_id":"authority.n.02"},{"index":3,"sense_key":"authority%1:18:00::","synset_id":"authority.n.03"},{"index":4,"sense_key":"authority%1:09:00::","synset_id":"assurance.n.01"},{"index":5,"sense_key":"authority%1:14:00::","synset_id":"agency.n.01"},{"index":6,"sense_key":"authority%1:07:02::","synset_id":"authority.n.06"},{"index":7,"sense_key":"authority%1:10:00::","synset_id":"authority.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"authority%1:14:00::","call_ids":["semeval2015.d002.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"authority%1:14:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034114999999999997,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.344096166023519}
{"item_id":"semeval2015.d002.s001.t012","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002096,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.615164334012661}
{"item_id":"semeval2015.d002.s001.t013","gold_sense_keys":["voluntary%3:00:01::"],"candidates":[{"index":1,"sense_key":"voluntary%3:00:01::","synset_id":"voluntary.a.01"},{"index":2,"sense_key":"voluntary%3:00:02::","synset_id":"voluntary.a.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"voluntary%3:00:01::","call_ids":["semeval2015.d002.s001.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"voluntary%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":308,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002778,"input_uncached_usd":0.000015399999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.937220874999184}
{"item_id":"semeval2015.d002.s001.t014","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":2,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":3,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":4,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":5,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":6,"sense_key":"community%1:14:02::","synset_id":"community.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"community%1:14:00::","call_ids":["semeval2015.d002.s001.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":1618,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006696999999999999,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.0006472,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.005032167013269}
{"item_id":"semeval2015.d002.s001.t015","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":2,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":3,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":4,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":5,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":6,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"organisation%1:14:00::","call_ids":["semeval2015.d002.s001.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"organisation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001647,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9911789590260014}
{"item_id":"semeval2015.d002.s001.t017","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"service%1:14:05::","call_ids":["semeval2015.d002.s001.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"service%1:14:05::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":809,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004820499999999999,"input_uncached_usd":0.00004045,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.218870875018183}
{"item_id":"semeval2015.d002.s001.t018","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d002.s001.t018__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":321,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00032964999999999996,"input_uncached_usd":0.00001605,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.470213874999899}
{"item_id":"semeval2015.d002.s002.t002","gold_sense_keys":["case%1:11:00::"],"candidates":[{"index":1,"sense_key":"case%1:11:00::","synset_id":"case.n.01"},{"index":2,"sense_key":"case%1:26:00::","synset_id":"event.n.02"},{"index":3,"sense_key":"case%1:04:00::","synset_id":"lawsuit.n.01"},{"index":4,"sense_key":"case%1:09:00::","synset_id":"case.n.04"},{"index":5,"sense_key":"case%1:06:00::","synset_id":"case.n.05"},{"index":6,"sense_key":"case%1:18:03::","synset_id":"case.n.06"},{"index":7,"sense_key":"case%1:18:02::","synset_id":"subject.n.06"},{"index":8,"sense_key":"case%1:10:02::","synset_id":"case.n.08"},{"index":9,"sense_key":"case%1:10:01::","synset_id":"case.n.09"},{"index":10,"sense_key":"case%1:23:00::","synset_id":"case.n.10"},{"index":11,"sense_key":"case%1:10:00::","synset_id":"case.n.11"},{"index":12,"sense_key":"case%1:26:02::","synset_id":"case.n.12"},{"index":13,"sense_key":"case%1:18:04::","synset_id":"character.n.05"},{"index":14,"sense_key":"case%1:10:03::","synset_id":"font.n.01"},{"index":15,"sense_key":"case%1:08:00::","synset_id":"sheath.n.02"},{"index":16,"sense_key":"case%1:06:06::","synset_id":"shell.n.08"},{"index":17,"sense_key":"case%1:06:07::","synset_id":"casing.n.03"},{"index":18,"sense_key":"case%1:06:03::","synset_id":"case.n.18"},{"index":19,"sense_key":"case%1:06:02::","synset_id":"case.n.19"},{"index":20,"sense_key":"case%1:06:01::","synset_id":"case.n.20"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"case%1:04:00::","call_ids":["semeval2015.d002.s002.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"case%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1106,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007529,"input_uncached_usd":0.000055299999999999996,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.028028167027514}
{"item_id":"semeval2015.d002.s002.t003","gold_sense_keys":["report%2:32:04::"],"candidates":[{"index":1,"sense_key":"report%2:32:00::","synset_id":"report.v.01"},{"index":2,"sense_key":"report%2:32:04::","synset_id":"report.v.02"},{"index":3,"sense_key":"report%2:32:03::","synset_id":"report.v.03"},{"index":4,"sense_key":"report%2:32:01::","synset_id":"report.v.04"},{"index":5,"sense_key":"report%2:32:13::","synset_id":"report.v.05"},{"index":6,"sense_key":"report%2:32:02::","synset_id":"report.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"report%2:32:04::","call_ids":["semeval2015.d002.s002.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"report%2:32:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":499,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005433500000000001,"input_uncached_usd":0.00002495,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.913592291995883}
{"item_id":"semeval2015.d002.s002.t004","gold_sense_keys":["attempt%1:04:00::"],"candidates":[{"index":1,"sense_key":"attempt%1:04:00::","synset_id":"attempt.n.01"},{"index":2,"sense_key":"attempt%1:04:02::","synset_id":"attack.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attempt%1:04:00::","call_ids":["semeval2015.d002.s002.t004__v1__a1","semeval2015.d002.s002.t004__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attempt%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":739,"cached_input_tokens":0,"output_tokens":1120,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00048494999999999994,"input_uncached_usd":0.000036950000000000004,"input_cached_usd":0.0,"output_usd":0.00044799999999999994,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.852120500989258}
{"item_id":"semeval2015.d002.s002.t005","gold_sense_keys":["involve%2:42:04::"],"candidates":[{"index":1,"sense_key":"involve%2:42:01::","synset_id":"involve.v.01"},{"index":2,"sense_key":"involve%2:42:04::","synset_id":"involve.v.02"},{"index":3,"sense_key":"involve%2:42:00::","synset_id":"imply.v.05"},{"index":4,"sense_key":"involve%2:42:07::","synset_id":"necessitate.v.01"},{"index":5,"sense_key":"involve%2:42:02::","synset_id":"involve.v.05"},{"index":6,"sense_key":"involve%2:31:00::","synset_id":"involve.v.06"},{"index":7,"sense_key":"involve%2:30:00::","synset_id":"involve.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"involve%2:42:04::","call_ids":["semeval2015.d002.s002.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"involve%2:42:04::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":511,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031355,"input_uncached_usd":0.000025549999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.832137959019747}
{"item_id":"semeval2015.d002.s002.t006","gold_sense_keys":["user%1:18:00::"],"candidates":[{"index":1,"sense_key":"user%1:18:00::","synset_id":"user.n.01"},{"index":2,"sense_key":"user%1:18:01::","synset_id":"exploiter.n.01"},{"index":3,"sense_key":"user%1:18:02::","synset_id":"drug_user.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"user%1:18:00::","call_ids":["semeval2015.d002.s002.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"user%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":337,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00012565,"input_uncached_usd":0.00001685,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.45030762499664}
{"item_id":"semeval2015.d002.s002.t007","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s002.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008156000000000001,"input_uncached_usd":0.0000412,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.304716416983865}
{"item_id":"semeval2015.d002.s002.t008","gold_sense_keys":["planning%1:04:00::","planning%1:04:02::"],"candidates":[{"index":1,"sense_key":"planning%1:04:00::","synset_id":"planning.n.01"},{"index":2,"sense_key":"planning%1:04:02::","synset_id":"planning.n.02"},{"index":3,"sense_key":"planning%1:09:00::","synset_id":"planning.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"planning%1:04:02::","call_ids":["semeval2015.d002.s002.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"planning%1:04:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":371,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028094999999999997,"input_uncached_usd":0.00001855,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.516283625038341}
{"item_id":"semeval2015.d002.s002.t009","gold_sense_keys":["involvement%1:04:00::"],"candidates":[{"index":1,"sense_key":"involvement%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"involvement%1:24:00::","synset_id":"involvement.n.02"},{"index":3,"sense_key":"involvement%1:09:00::","synset_id":"interest.n.01"},{"index":4,"sense_key":"involvement%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"involvement%1:26:01::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"involvement%1:04:00::","call_ids":["semeval2015.d002.s002.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"involvement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033644999999999997,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.4037338749621995}
{"item_id":"semeval2015.d002.s002.t010","gold_sense_keys":["service%1:04:08::"],"candidates":[{"index":1,"sense_key":"service%1:04:08::","synset_id":"service.n.01"},{"index":2,"sense_key":"service%1:04:00::","synset_id":"service.n.02"},{"index":3,"sense_key":"service%1:04:01::","synset_id":"service.n.03"},{"index":4,"sense_key":"service%1:14:05::","synset_id":"service.n.04"},{"index":5,"sense_key":"service%1:04:07::","synset_id":"service.n.05"},{"index":6,"sense_key":"service%1:14:00::","synset_id":"military_service.n.01"},{"index":7,"sense_key":"service%1:18:00::","synset_id":"service.n.07"},{"index":8,"sense_key":"service%1:07:00::","synset_id":"avail.n.01"},{"index":9,"sense_key":"service%1:06:00::","synset_id":"service.n.09"},{"index":10,"sense_key":"service%1:04:03::","synset_id":"servicing.n.01"},{"index":11,"sense_key":"service%1:04:09::","synset_id":"service.n.11"},{"index":12,"sense_key":"service%1:04:02::","synset_id":"serve.n.01"},{"index":13,"sense_key":"service%1:04:04::","synset_id":"service.n.13"},{"index":14,"sense_key":"service%1:04:05::","synset_id":"overhaul.n.01"},{"index":15,"sense_key":"service%1:04:06::","synset_id":"service.n.15"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"service%1:04:08::","call_ids":["semeval2015.d002.s002.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"service%1:04:08::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":824,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006108000000000001,"input_uncached_usd":0.0000412,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.848405291966628}
{"item_id":"semeval2015.d002.s002.t011","gold_sense_keys":["provision%1:04:01::"],"candidates":[{"index":1,"sense_key":"provision%1:10:00::","synset_id":"provision.n.01"},{"index":2,"sense_key":"provision%1:04:01::","synset_id":"provision.n.02"},{"index":3,"sense_key":"provision%1:09:00::","synset_id":"planning.n.03"},{"index":4,"sense_key":"provision%1:21:00::","synset_id":"provision.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"provision%1:04:01::","call_ids":["semeval2015.d002.s002.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"provision%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003077,"input_uncached_usd":0.000019699999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.094524040992837}
{"item_id":"semeval2015.d002.s002.t012","gold_sense_keys":["find%2:32:00::","find%2:39:02::"],"candidates":[{"index":1,"sense_key":"find%2:40:02::","synset_id":"find.v.01"},{"index":2,"sense_key":"find%2:39:02::","synset_id":"detect.v.01"},{"index":3,"sense_key":"find%2:40:00::","synset_id":"find.v.03"},{"index":4,"sense_key":"find%2:32:00::","synset_id":"determine.v.01"},{"index":5,"sense_key":"find%2:31:10::","synset_id":"find.v.05"},{"index":6,"sense_key":"find%2:39:01::","synset_id":"witness.v.02"},{"index":7,"sense_key":"find%2:40:01::","synset_id":"line_up.v.02"},{"index":8,"sense_key":"find%2:36:00::","synset_id":"discover.v.03"},{"index":9,"sense_key":"find%2:31:09::","synset_id":"discover.v.04"},{"index":10,"sense_key":"find%2:40:03::","synset_id":"find.v.10"},{"index":11,"sense_key":"find%2:32:01::","synset_id":"rule.v.04"},{"index":12,"sense_key":"find%2:30:00::","synset_id":"receive.v.02"},{"index":13,"sense_key":"find%2:39:05::","synset_id":"find.v.13"},{"index":14,"sense_key":"find%2:40:15::","synset_id":"recover.v.01"},{"index":15,"sense_key":"find%2:38:10::","synset_id":"find.v.15"},{"index":16,"sense_key":"find%2:30:13::","synset_id":"find_oneself.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"find%2:32:00::","call_ids":["semeval2015.d002.s002.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"find%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1036,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00039099999999999996,"input_uncached_usd":0.0000518,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.847498083021492}
{"item_id":"semeval2015.d002.s002.t013","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["semeval2015.d002.s002.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00030445,"input_uncached_usd":0.00001645,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.409971375018358}
{"item_id":"semeval2015.d002.s002.t014","gold_sense_keys":["common%3:00:01::","common%5:00:00:familiar:02"],"candidates":[{"index":1,"sense_key":"common%3:00:02::","synset_id":"common.a.01"},{"index":2,"sense_key":"common%3:00:01::","synset_id":"common.a.02"},{"index":3,"sense_key":"common%5:00:00:shared:00","synset_id":"common.s.01"},{"index":4,"sense_key":"common%5:00:00:familiar:02","synset_id":"common.s.02"},{"index":5,"sense_key":"common%5:00:00:informal:02","synset_id":"common.s.03"},{"index":6,"sense_key":"common%5:00:00:lowborn:00","synset_id":"common.s.04"},{"index":7,"sense_key":"common%5:00:00:inferior:02","synset_id":"coarse.s.02"},{"index":8,"sense_key":"common%5:00:00:unrefined:01","synset_id":"coarse.s.01"},{"index":9,"sense_key":"common%5:00:00:ordinary:00","synset_id":"common.s.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"common%3:00:01::","call_ids":["semeval2015.d002.s002.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"common%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":684,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.0005013999999999999,"input_uncached_usd":0.0000342,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.694964458001778}
{"item_id":"semeval2015.d002.s003.t002","gold_sense_keys":["variety%1:07:00::","variety%1:14:01::"],"candidates":[{"index":1,"sense_key":"variety%1:14:01::","synset_id":"assortment.n.01"},{"index":2,"sense_key":"variety%1:07:00::","synset_id":"diverseness.n.01"},{"index":3,"sense_key":"variety%1:14:00::","synset_id":"variety.n.03"},{"index":4,"sense_key":"variety%1:10:00::","synset_id":"variety_show.n.01"},{"index":5,"sense_key":"variety%1:09:01::","synset_id":"kind.n.01"},{"index":6,"sense_key":"variety%1:07:01::","synset_id":"variety.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"variety%1:07:00::","call_ids":["semeval2015.d002.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"variety%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":585,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018925,"input_uncached_usd":0.00002925,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.516564332996495}
{"item_id":"semeval2015.d002.s003.t003","gold_sense_keys":["different%3:00:02::","different%5:00:01:other:00"],"candidates":[{"index":1,"sense_key":"different%3:00:00::","synset_id":"different.a.01"},{"index":2,"sense_key":"different%5:00:00:other:00","synset_id":"different.s.01"},{"index":3,"sense_key":"different%5:00:00:unusual:00","synset_id":"different.s.02"},{"index":4,"sense_key":"different%3:00:02::","synset_id":"unlike.a.01"},{"index":5,"sense_key":"different%5:00:01:other:00","synset_id":"different.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"different%3:00:02::","call_ids":["semeval2015.d002.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"different%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0006965,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.249629124999046}
{"item_id":"semeval2015.d002.s003.t005","gold_sense_keys":["play%2:41:12::"],"candidates":[{"index":1,"sense_key":"play%2:33:00::","synset_id":"play.v.01"},{"index":2,"sense_key":"play%2:41:12::","synset_id":"play.v.02"},{"index":3,"sense_key":"play%2:36:12::","synset_id":"play.v.03"},{"index":4,"sense_key":"play%2:36:02::","synset_id":"act.v.03"},{"index":5,"sense_key":"play%2:41:03::","synset_id":"play.v.05"},{"index":6,"sense_key":"play%2:36:00::","synset_id":"play.v.06"},{"index":7,"sense_key":"play%2:36:01::","synset_id":"play.v.07"},{"index":8,"sense_key":"play%2:29:01::","synset_id":"act.v.05"},{"index":9,"sense_key":"play%2:38:00::","synset_id":"play.v.09"},{"index":10,"sense_key":"play%2:33:01::","synset_id":"play.v.10"},{"index":11,"sense_key":"play%2:41:00::","synset_id":"play.v.11"},{"index":12,"sense_key":"play%2:36:06::","synset_id":"play.v.12"},{"index":13,"sense_key":"play%2:36:11::","synset_id":"play.v.13"},{"index":14,"sense_key":"play%2:36:03::","synset_id":"play.v.14"},{"index":15,"sense_key":"play%2:33:03::","synset_id":"play.v.15"},{"index":16,"sense_key":"play%2:29:00::","synset_id":"play.v.16"},{"index":17,"sense_key":"play%2:41:02::","synset_id":"play.v.17"},{"index":18,"sense_key":"play%2:36:05::","synset_id":"play.v.18"},{"index":19,"sense_key":"play%2:35:00::","synset_id":"toy.v.02"},{"index":20,"sense_key":"play%2:34:13::","synset_id":"play.v.20"},{"index":21,"sense_key":"play%2:31:01::","synset_id":"dally.v.04"},{"index":22,"sense_key":"play%2:42:00::","synset_id":"play.v.22"},{"index":23,"sense_key":"play%2:41:13::","synset_id":"dally.v.01"},{"index":24,"sense_key":"play%2:38:03::","synset_id":"play.v.24"},{"index":25,"sense_key":"play%2:36:10::","synset_id":"act.v.10"},{"index":26,"sense_key":"play%2:36:04::","synset_id":"play.v.26"},{"index":27,"sense_key":"play%2:36:13::","synset_id":"bring.v.03"},{"index":28,"sense_key":"play%2:35:07::","synset_id":"play.v.28"},{"index":29,"sense_key":"play%2:33:07::","synset_id":"play.v.29"},{"index":30,"sense_key":"play%2:33:08::","synset_id":"bet.v.02"},{"index":31,"sense_key":"play%2:33:09::","synset_id":"play.v.31"},{"index":32,"sense_key":"play%2:33:13::","synset_id":"play.v.32"},{"index":33,"sense_key":"play%2:33:12::","synset_id":"play.v.33"},{"index":34,"sense_key":"play%2:33:02::","synset_id":"meet.v.10"},{"index":35,"sense_key":"play%2:29:04::","synset_id":"play.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"play%2:36:02::","call_ids":["semeval2015.d002.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"play%2:36:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1818,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002765,"input_uncached_usd":0.0000909,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0528476249892265}
{"item_id":"semeval2015.d002.s003.t006","gold_sense_keys":["public%5:00:00:common:02"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d002.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":357,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006130500000000001,"input_uncached_usd":0.00001785,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.611313625006005}
{"item_id":"semeval2015.d002.s003.t007","gold_sense_keys":["welfare%1:04:00::"],"candidates":[{"index":1,"sense_key":"welfare%1:04:00::","synset_id":"social_welfare.n.01"},{"index":2,"sense_key":"welfare%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"welfare%1:26:00::","synset_id":"wellbeing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"welfare%1:04:00::","call_ids":["semeval2015.d002.s003.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"welfare%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002577,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.451301791996229}
{"item_id":"semeval2015.d002.s004.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039664999999999996,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.886529999959748}
{"item_id":"semeval2015.d002.s004.t002","gold_sense_keys":["commission%1:14:00::","commission%1:14:01::"],"candidates":[{"index":1,"sense_key":"commission%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"commission%1:21:00::","synset_id":"commission.n.02"},{"index":3,"sense_key":"commission%1:04:02::","synset_id":"commission.n.03"},{"index":4,"sense_key":"commission%1:26:00::","synset_id":"commission.n.04"},{"index":5,"sense_key":"commission%1:14:01::","synset_id":"deputation.n.01"},{"index":6,"sense_key":"commission%1:10:01::","synset_id":"commission.n.06"},{"index":7,"sense_key":"commission%1:10:00::","synset_id":"commission.n.07"},{"index":8,"sense_key":"commission%1:04:01::","synset_id":"perpetration.n.01"},{"index":9,"sense_key":"commission%1:04:00::","synset_id":"mission.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"commission%1:14:01::","call_ids":["semeval2015.d002.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"commission%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":686,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006551,"input_uncached_usd":0.0000343,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.316832833981607}
{"item_id":"semeval2015.d002.s004.t004","gold_sense_keys":["ministry%1:14:00::"],"candidates":[{"index":1,"sense_key":"ministry%1:14:01::","synset_id":"ministry.n.01"},{"index":2,"sense_key":"ministry%1:06:00::","synset_id":"ministry.n.02"},{"index":3,"sense_key":"ministry%1:14:00::","synset_id":"ministry.n.03"},{"index":4,"sense_key":"ministry%1:04:00::","synset_id":"ministry.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"ministry%1:14:00::","call_ids":["semeval2015.d002.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"ministry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":423,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015554999999999999,"input_uncached_usd":0.00002115,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.594723707996309}
{"item_id":"semeval2015.d002.s004.t005","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":1936,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.0008025500000000001,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0007744,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.744262333028018}
{"item_id":"semeval2015.d002.s004.t006","gold_sense_keys":["affairs%1:04:00::"],"candidates":[{"index":1,"sense_key":"affairs%1:09:00::","synset_id":"personal_business.n.01"},{"index":2,"sense_key":"affairs%1:04:00::","synset_id":"affairs.n.02"},{"index":3,"sense_key":"matter%1:09:01::","synset_id":"matter.n.01"},{"index":4,"sense_key":"affair%1:26:00::","synset_id":"affair.n.02"},{"index":5,"sense_key":"affair%1:11:00::","synset_id":"affair.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"affairs%1:04:00::","call_ids":["semeval2015.d002.s004.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"affairs%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":525,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028905,"input_uncached_usd":0.000026249999999999998,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.481110666994937}
{"item_id":"semeval2015.d002.s004.t007","gold_sense_keys":["regional%5:00:00:territorial:00"],"candidates":[{"index":1,"sense_key":"regional%3:01:00::","synset_id":"regional.a.01"},{"index":2,"sense_key":"regional%5:00:00:territorial:00","synset_id":"regional.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"regional%5:00:00:territorial:00","call_ids":["semeval2015.d002.s004.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"regional%5:00:00:territorial:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":372,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005114000000000001,"input_uncached_usd":0.000018599999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.142953667032998}
{"item_id":"semeval2015.d002.s004.t008","gold_sense_keys":["government%1:14:00::"],"candidates":[{"index":1,"sense_key":"government%1:14:00::","synset_id":"government.n.01"},{"index":2,"sense_key":"government%1:04:00::","synset_id":"government.n.02"},{"index":3,"sense_key":"government%1:09:00::","synset_id":"government.n.03"},{"index":4,"sense_key":"government%1:09:01::","synset_id":"politics.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"government%1:14:00::","call_ids":["semeval2015.d002.s004.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"government%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001846,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2595702919643372}
{"item_id":"semeval2015.d002.s004.t010","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s004.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":414,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025749999999999997,"input_uncached_usd":0.0000207,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.057926542009227}
{"item_id":"semeval2015.d002.s004.t011","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s004.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034175,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.984632417035755}
{"item_id":"semeval2015.d002.s004.t012","gold_sense_keys":["exclusion%1:26:00::"],"candidates":[{"index":1,"sense_key":"exclusion%1:26:00::","synset_id":"exclusion.n.01"},{"index":2,"sense_key":"exclusion%1:26:01::","synset_id":"excommunication.n.01"},{"index":3,"sense_key":"exclusion%1:09:00::","synset_id":"exception.n.01"},{"index":4,"sense_key":"exclusion%1:04:01::","synset_id":"ejection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"exclusion%1:09:00::","call_ids":["semeval2015.d002.s004.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"exclusion%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049095,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.621504915994592}
{"item_id":"semeval2015.d002.s004.t013","gold_sense_keys":["major%3:00:02::","major%3:00:06::"],"candidates":[{"index":1,"sense_key":"major%3:00:06::","synset_id":"major.a.01"},{"index":2,"sense_key":"major%3:00:02::","synset_id":"major.a.02"},{"index":3,"sense_key":"major%3:00:01::","synset_id":"major.a.03"},{"index":4,"sense_key":"major%3:00:05::","synset_id":"major.a.04"},{"index":5,"sense_key":"major%3:00:04::","synset_id":"major.a.05"},{"index":6,"sense_key":"major%3:00:07::","synset_id":"major.a.06"},{"index":7,"sense_key":"major%3:00:03::","synset_id":"major.a.07"},{"index":8,"sense_key":"major%5:00:00:senior:00","synset_id":"major.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"major%3:00:02::","call_ids":["semeval2015.d002.s004.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"major%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":575,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00077755,"input_uncached_usd":0.000028749999999999997,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.001341042050626}
{"item_id":"semeval2015.d002.s004.t014","gold_sense_keys":["challenge%1:26:00::"],"candidates":[{"index":1,"sense_key":"challenge%1:26:00::","synset_id":"challenge.n.01"},{"index":2,"sense_key":"challenge%1:10:00::","synset_id":"challenge.n.02"},{"index":3,"sense_key":"challenge%1:10:01::","synset_id":"challenge.n.03"},{"index":4,"sense_key":"challenge%1:10:02::","synset_id":"challenge.n.04"},{"index":5,"sense_key":"challenge%1:10:03::","synset_id":"challenge.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"challenge%1:26:00::","call_ids":["semeval2015.d002.s004.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"challenge%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":467,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018334999999999998,"input_uncached_usd":0.00002335,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.836203541024588}
{"item_id":"semeval2015.d002.s004.t015","gold_sense_keys":["public%5:00:00:common:02"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d002.s004.t015__v1__a1","semeval2015.d002.s004.t015__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":2016,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.0008475500000000001,"input_uncached_usd":0.00004115,"input_cached_usd":0.0,"output_usd":0.0008064,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.125209209043533}
{"item_id":"semeval2015.d002.s004.t016","gold_sense_keys":["welfare%1:04:00::"],"candidates":[{"index":1,"sense_key":"welfare%1:04:00::","synset_id":"social_welfare.n.01"},{"index":2,"sense_key":"welfare%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"welfare%1:26:00::","synset_id":"wellbeing.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"welfare%1:04:00::","call_ids":["semeval2015.d002.s004.t016__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"welfare%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023365,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5863221249892376}
{"item_id":"semeval2015.d002.s005.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":665,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032125000000000003,"input_uncached_usd":0.00003325,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.508933999983128}
{"item_id":"semeval2015.d002.s005.t002","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d002.s005.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.000287,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.313542667019647}
{"item_id":"semeval2015.d002.s005.t003","gold_sense_keys":["comparative%3:01:00::"],"candidates":[{"index":1,"sense_key":"comparative%3:01:00::","synset_id":"comparative.a.01"},{"index":2,"sense_key":"comparative%3:00:00::","synset_id":"relative.a.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"comparative%3:01:00::","call_ids":["semeval2015.d002.s005.t003__v1__a1","semeval2015.d002.s005.t003__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"comparative%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":736,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00033774999999999994,"input_uncached_usd":0.000043349999999999997,"input_cached_usd":0.0,"output_usd":0.0002944,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.971021873992868}
{"item_id":"semeval2015.d002.s005.t004","gold_sense_keys":["study%1:04:00::","study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":754,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002489,"input_uncached_usd":0.0000377,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4804695419734344}
{"item_id":"semeval2015.d002.s005.t005","gold_sense_keys":["part%1:09:00::","part%1:24:00::"],"candidates":[{"index":1,"sense_key":"part%1:24:00::","synset_id":"part.n.01"},{"index":2,"sense_key":"part%1:06:00::","synset_id":"part.n.02"},{"index":3,"sense_key":"part%1:17:00::","synset_id":"part.n.03"},{"index":4,"sense_key":"part%1:09:02::","synset_id":"part.n.04"},{"index":5,"sense_key":"part%1:15:00::","synset_id":"region.n.01"},{"index":6,"sense_key":"part%1:04:01::","synset_id":"function.n.03"},{"index":7,"sense_key":"part%1:09:01::","synset_id":"character.n.04"},{"index":8,"sense_key":"part%1:21:00::","synset_id":"share.n.01"},{"index":9,"sense_key":"part%1:09:00::","synset_id":"part.n.09"},{"index":10,"sense_key":"part%1:08:01::","synset_id":"part.n.10"},{"index":11,"sense_key":"part%1:10:00::","synset_id":"part.n.11"},{"index":12,"sense_key":"part%1:04:03::","synset_id":"contribution.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"part%1:24:00::","call_ids":["semeval2015.d002.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"part%1:24:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":933,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00051425,"input_uncached_usd":0.000046649999999999996,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.486257750017103}
{"item_id":"semeval2015.d002.s005.t007","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":520,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001604,"input_uncached_usd":0.000026,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1674015420139767}
{"item_id":"semeval2015.d002.s005.t008","gold_sense_keys":["extent%1:26:00::"],"candidates":[{"index":1,"sense_key":"extent%1:26:00::","synset_id":"extent.n.01"},{"index":2,"sense_key":"extent%1:07:00::","synset_id":"extent.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extent%1:26:00::","call_ids":["semeval2015.d002.s005.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extent%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":422,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005395,"input_uncached_usd":0.000021099999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.45760462502949}
{"item_id":"semeval2015.d002.s005.t009","gold_sense_keys":["direct%5:00:00:immediate:00"],"candidates":[{"index":1,"sense_key":"direct%3:00:00::","synset_id":"direct.a.01"},{"index":2,"sense_key":"direct%5:00:00:immediate:00","synset_id":"direct.s.01"},{"index":3,"sense_key":"direct%3:00:02::","synset_id":"direct.a.03"},{"index":4,"sense_key":"direct%3:00:04::","synset_id":"lineal.a.01"},{"index":5,"sense_key":"direct%3:00:05::","synset_id":"direct.a.05"},{"index":6,"sense_key":"direct%3:00:03::","synset_id":"direct.a.06"},{"index":7,"sense_key":"direct%3:00:01::","synset_id":"direct.a.07"},{"index":8,"sense_key":"direct%5:00:00:primary:00","synset_id":"direct.s.02"},{"index":9,"sense_key":"direct%5:00:00:exact:00","synset_id":"direct.s.03"},{"index":10,"sense_key":"direct%5:00:00:absolute:00","synset_id":"direct.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"direct%5:00:00:immediate:00","call_ids":["semeval2015.d002.s005.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"direct%5:00:00:immediate:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":744,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00040199999999999996,"input_uncached_usd":0.000037199999999999996,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.052698332990985}
{"item_id":"semeval2015.d002.s005.t010","gold_sense_keys":["participation%1:04:00::"],"candidates":[{"index":1,"sense_key":"participation%1:04:00::","synset_id":"engagement.n.07"},{"index":2,"sense_key":"participation%1:26:00::","synset_id":"participation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"participation%1:04:00::","call_ids":["semeval2015.d002.s005.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"participation%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":432,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.000668,"input_uncached_usd":0.0000216,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.550125709036365}
{"item_id":"semeval2015.d002.s005.t012","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"work%1:04:00::","call_ids":["semeval2015.d002.s005.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"work%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029785,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.262649124953896}
{"item_id":"semeval2015.d002.s005.t013","gold_sense_keys":["increase%2:30:02::"],"candidates":[{"index":1,"sense_key":"increase%2:30:00::","synset_id":"increase.v.01"},{"index":2,"sense_key":"increase%2:30:02::","synset_id":"increase.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"increase%2:30:00::","call_ids":["semeval2015.d002.s005.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"increase%2:30:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":1233,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005138499999999999,"input_uncached_usd":0.00002065,"input_cached_usd":0.0,"output_usd":0.0004932,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.259629915992264}
{"item_id":"semeval2015.d002.s006.t000","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d002.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":753,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00017204999999999998,"input_uncached_usd":0.00003765,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.612086042005103}
{"item_id":"semeval2015.d002.s006.t001","gold_sense_keys":["base%2:31:00::"],"candidates":[{"index":1,"sense_key":"base%2:31:00::","synset_id":"establish.v.08"},{"index":2,"sense_key":"base%2:42:04::","synset_id":"base.v.02"},{"index":3,"sense_key":"base%2:34:00::","synset_id":"free-base.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"base%2:31:00::","call_ids":["semeval2015.d002.s006.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"base%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005411,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.00006937503349}
{"item_id":"semeval2015.d002.s006.t002","gold_sense_keys":["personal%3:00:00::","personal%5:00:00:subjective:00"],"candidates":[{"index":1,"sense_key":"personal%3:00:00::","synset_id":"personal.a.01"},{"index":2,"sense_key":"personal%5:00:00:subjective:00","synset_id":"personal.s.01"},{"index":3,"sense_key":"personal%3:01:01::","synset_id":"personal.a.03"},{"index":4,"sense_key":"personal%5:00:00:physical:00","synset_id":"personal.s.02"},{"index":5,"sense_key":"personal%3:01:00::","synset_id":"personal.a.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"personal%5:00:00:subjective:00","call_ids":["semeval2015.d002.s006.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"personal%5:00:00:subjective:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":496,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003384,"input_uncached_usd":0.0000248,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.164159790962003}
{"item_id":"semeval2015.d002.s006.t003","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"},{"index":2,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2015.d002.s006.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002588,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.916718791995663}
{"item_id":"semeval2015.d002.s006.t004","gold_sense_keys":["senior%3:00:00::"],"candidates":[{"index":1,"sense_key":"senior%3:00:00::","synset_id":"senior.a.01"},{"index":2,"sense_key":"senior%5:00:00:last:00","synset_id":"senior.s.01"},{"index":3,"sense_key":"senior%5:00:00:old:02","synset_id":"aged.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"senior%3:00:00::","call_ids":["semeval2015.d002.s006.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"senior%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015869999999999998,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.93960533401696}
{"item_id":"semeval2015.d002.s006.t005","gold_sense_keys":["official%1:18:01::"],"candidates":[{"index":1,"sense_key":"official%1:18:01::","synset_id":"official.n.01"},{"index":2,"sense_key":"official%1:18:00::","synset_id":"official.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"official%1:18:01::","call_ids":["semeval2015.d002.s006.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"official%1:18:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":413,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023184999999999997,"input_uncached_usd":0.00002065,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.371480915986467}
{"item_id":"semeval2015.d002.s006.t006","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s006.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":590,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002919,"input_uncached_usd":0.0000295,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.019600624975283}
{"item_id":"semeval2015.d002.s006.t007","gold_sense_keys":["representative%1:18:00::","representative%1:18:02::"],"candidates":[{"index":1,"sense_key":"representative%1:18:00::","synset_id":"representative.n.01"},{"index":2,"sense_key":"representative%1:18:02::","synset_id":"spokesperson.n.01"},{"index":3,"sense_key":"representative%1:18:01::","synset_id":"congressman.n.01"},{"index":4,"sense_key":"representative%1:09:00::","synset_id":"example.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"representative%1:18:00::","call_ids":["semeval2015.d002.s006.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"representative%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026225,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.3901528749847785}
{"item_id":"semeval2015.d002.s006.t008","gold_sense_keys":["national%3:00:01::","national%3:01:00::"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["semeval2015.d002.s006.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001391,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.240856165997684}
{"item_id":"semeval2015.d002.s006.t009","gold_sense_keys":["level%1:26:01::"],"candidates":[{"index":1,"sense_key":"level%1:07:00::","synset_id":"degree.n.01"},{"index":2,"sense_key":"level%1:26:01::","synset_id":"grade.n.02"},{"index":3,"sense_key":"level%1:26:00::","synset_id":"degree.n.02"},{"index":4,"sense_key":"level%1:07:01::","synset_id":"level.n.04"},{"index":5,"sense_key":"level%1:06:00::","synset_id":"level.n.05"},{"index":6,"sense_key":"level%1:06:03::","synset_id":"horizontal_surface.n.01"},{"index":7,"sense_key":"level%1:09:00::","synset_id":"level.n.07"},{"index":8,"sense_key":"level%1:06:01::","synset_id":"floor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"level%1:26:00::","call_ids":["semeval2015.d002.s006.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"level%1:26:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":701,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037464999999999997,"input_uncached_usd":0.00003505,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.635147541004699}
{"item_id":"semeval2015.d002.s006.t010","gold_sense_keys":["organisation%1:14:00::","organisation%1:14:01::"],"candidates":[{"index":1,"sense_key":"organisation%1:14:01::","synset_id":"administration.n.02"},{"index":2,"sense_key":"organisation%1:14:00::","synset_id":"organization.n.01"},{"index":3,"sense_key":"organisation%1:09:00::","synset_id":"arrangement.n.03"},{"index":4,"sense_key":"organisation%1:07:00::","synset_id":"organization.n.05"},{"index":5,"sense_key":"organisation%1:04:00::","synset_id":"organization.n.04"},{"index":6,"sense_key":"organisation%1:04:01::","synset_id":"organization.n.06"},{"index":7,"sense_key":"organisation%1:04:02::","synset_id":"constitution.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"organisation%1:09:00::","call_ids":["semeval2015.d002.s006.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"organisation%1:09:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":716,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005286000000000001,"input_uncached_usd":0.000035799999999999996,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.655381499964278}
{"item_id":"semeval2015.d002.s006.t013","gold_sense_keys":["country%1:15:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"country%1:15:00::","call_ids":["semeval2015.d002.s006.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"country%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":587,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029174999999999996,"input_uncached_usd":0.00002935,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.251986458024476}
{"item_id":"semeval2015.d002.s007.t000","gold_sense_keys":["important%3:00:00::","important%3:00:02::","important%3:00:04::"],"candidates":[{"index":1,"sense_key":"important%3:00:00::","synset_id":"important.a.01"},{"index":2,"sense_key":"important%3:00:04::","synset_id":"significant.a.01"},{"index":3,"sense_key":"important%3:00:02::","synset_id":"crucial.a.01"},{"index":4,"sense_key":"important%5:00:00:influential:00","synset_id":"authoritative.s.01"},{"index":5,"sense_key":"important%5:00:00:immodest:02","synset_id":"important.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"important%3:00:00::","call_ids":["semeval2015.d002.s007.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"important%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":596,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006762,"input_uncached_usd":0.0000298,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.855863708013203}
{"item_id":"semeval2015.d002.s007.t001","gold_sense_keys":["industrial%3:01:00::"],"candidates":[{"index":1,"sense_key":"industrial%3:01:00::","synset_id":"industrial.a.01"},{"index":2,"sense_key":"industrial%3:00:00::","synset_id":"industrial.a.02"},{"index":3,"sense_key":"industrial%5:00:00:blue-collar:00","synset_id":"industrial.s.01"},{"index":4,"sense_key":"industrial%5:00:00:heavy-duty:00","synset_id":"industrial.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"industrial%5:00:00:blue-collar:00","call_ids":["semeval2015.d002.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"industrial%5:00:00:blue-collar:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":500,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006458,"input_uncached_usd":0.000024999999999999998,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.280986208992545}
{"item_id":"semeval2015.d002.s007.t002","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034199999999999996,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.568943332997151}
{"item_id":"semeval2015.d002.s007.t003","gold_sense_keys":["metal%1:27:00::"],"candidates":[{"index":1,"sense_key":"metal%1:27:00::","synset_id":"metallic_element.n.01"},{"index":2,"sense_key":"metal%1:27:01::","synset_id":"alloy.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"metal%1:27:00::","call_ids":["semeval2015.d002.s007.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"metal%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":1744,"reasoning_output_tokens":1728},"cost":{"currency":"USD","total_usd":0.0007214000000000001,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.0006975999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.811937084014062}
{"item_id":"semeval2015.d002.s007.t004","gold_sense_keys":["industry%1:14:00::"],"candidates":[{"index":1,"sense_key":"industry%1:14:00::","synset_id":"industry.n.01"},{"index":2,"sense_key":"industry%1:04:00::","synset_id":"industry.n.02"},{"index":3,"sense_key":"industry%1:07:00::","synset_id":"diligence.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industry%1:14:00::","call_ids":["semeval2015.d002.s007.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industry%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":512,"cached_input_tokens":0,"output_tokens":1106,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.000468,"input_uncached_usd":0.0000256,"input_cached_usd":0.0,"output_usd":0.00044239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.901409333979245}
{"item_id":"semeval2015.d002.s007.t005","gold_sense_keys":["banking%1:04:01::"],"candidates":[{"index":1,"sense_key":"banking%1:04:01::","synset_id":"banking.n.01"},{"index":2,"sense_key":"banking%1:04:00::","synset_id":"banking.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"banking%1:04:01::","call_ids":["semeval2015.d002.s007.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"banking%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054075,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.873432207969017}
{"item_id":"semeval2015.d002.s007.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s007.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029079999999999997,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8557075830176473}
{"item_id":"semeval2015.d002.s007.t008","gold_sense_keys":["include%2:30:00::","include%2:31:00::"],"candidates":[{"index":1,"sense_key":"include%2:42:00::","synset_id":"include.v.01"},{"index":2,"sense_key":"include%2:31:00::","synset_id":"include.v.02"},{"index":3,"sense_key":"include%2:30:00::","synset_id":"include.v.03"},{"index":4,"sense_key":"include%2:41:03::","synset_id":"admit.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"include%2:30:00::","call_ids":["semeval2015.d002.s007.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"include%2:30:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006742,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.75016933400184}
{"item_id":"semeval2015.d002.s008.t000","gold_sense_keys":["speaker%1:18:00::"],"candidates":[{"index":1,"sense_key":"speaker%1:18:00::","synset_id":"speaker.n.01"},{"index":2,"sense_key":"speaker%1:06:00::","synset_id":"loudspeaker.n.01"},{"index":3,"sense_key":"speaker%1:18:01::","synset_id":"speaker.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"speaker%1:18:00::","call_ids":["semeval2015.d002.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"speaker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023874999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.649604041012935}
{"item_id":"semeval2015.d002.s008.t001","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034544999999999997,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.796794875001069}
{"item_id":"semeval2015.d002.s008.t002","gold_sense_keys":["exclusion%1:26:00::"],"candidates":[{"index":1,"sense_key":"exclusion%1:26:00::","synset_id":"exclusion.n.01"},{"index":2,"sense_key":"exclusion%1:26:01::","synset_id":"excommunication.n.01"},{"index":3,"sense_key":"exclusion%1:09:00::","synset_id":"exception.n.01"},{"index":4,"sense_key":"exclusion%1:04:01::","synset_id":"ejection.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"exclusion%1:26:00::","call_ids":["semeval2015.d002.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"exclusion%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003407,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.40243295795517}
{"item_id":"semeval2015.d002.s008.t003","gold_sense_keys":["conference%1:10:00::","conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s008.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":488,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003892,"input_uncached_usd":0.0000244,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.558156083978247}
{"item_id":"semeval2015.d002.s008.t004","gold_sense_keys":["left%1:15:00::"],"candidates":[{"index":1,"sense_key":"left%1:15:00::","synset_id":"left.n.01"},{"index":2,"sense_key":"left%1:14:00::","synset_id":"left.n.02"},{"index":3,"sense_key":"left%1:08:00::","synset_id":"left.n.03"},{"index":4,"sense_key":"left%1:06:00::","synset_id":"left_field.n.01"},{"index":5,"sense_key":"left%1:04:00::","synset_id":"left.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"left%1:15:00::","call_ids":["semeval2015.d002.s008.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"left%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":603,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039495,"input_uncached_usd":0.000030149999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.129364208027255}
{"item_id":"semeval2015.d002.s008.t005","gold_sense_keys":["right%1:15:00::"],"candidates":[{"index":1,"sense_key":"right%1:07:00::","synset_id":"right.n.01"},{"index":2,"sense_key":"right%1:15:00::","synset_id":"right.n.02"},{"index":3,"sense_key":"right%1:06:00::","synset_id":"right_field.n.01"},{"index":4,"sense_key":"right%1:14:00::","synset_id":"right.n.04"},{"index":5,"sense_key":"right%1:08:00::","synset_id":"right.n.05"},{"index":6,"sense_key":"right%1:04:00::","synset_id":"right.n.06"},{"index":7,"sense_key":"right%1:07:01::","synset_id":"right.n.07"},{"index":8,"sense_key":"right%1:21:00::","synset_id":"right.n.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"right%1:15:00::","call_ids":["semeval2015.d002.s008.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"right%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":775,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037794999999999994,"input_uncached_usd":0.00003875,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.93854016700061}
{"item_id":"semeval2015.d002.s008.t006","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"employment%1:04:00::","call_ids":["semeval2015.d002.s008.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"employment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":559,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.00075115,"input_uncached_usd":0.00002795,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.002315916994121}
{"item_id":"semeval2015.d002.s008.t007","gold_sense_keys":["industrial%3:01:00::"],"candidates":[{"index":1,"sense_key":"industrial%3:01:00::","synset_id":"industrial.a.01"},{"index":2,"sense_key":"industrial%3:00:00::","synset_id":"industrial.a.02"},{"index":3,"sense_key":"industrial%5:00:00:blue-collar:00","synset_id":"industrial.s.01"},{"index":4,"sense_key":"industrial%5:00:00:heavy-duty:00","synset_id":"industrial.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"industrial%3:01:00::","call_ids":["semeval2015.d002.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"industrial%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054415,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.935875291994307}
{"item_id":"semeval2015.d002.s008.t012","gold_sense_keys":["commission%1:14:00::","commission%1:14:01::"],"candidates":[{"index":1,"sense_key":"commission%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"commission%1:21:00::","synset_id":"commission.n.02"},{"index":3,"sense_key":"commission%1:04:02::","synset_id":"commission.n.03"},{"index":4,"sense_key":"commission%1:26:00::","synset_id":"commission.n.04"},{"index":5,"sense_key":"commission%1:14:01::","synset_id":"deputation.n.01"},{"index":6,"sense_key":"commission%1:10:01::","synset_id":"commission.n.06"},{"index":7,"sense_key":"commission%1:10:00::","synset_id":"commission.n.07"},{"index":8,"sense_key":"commission%1:04:01::","synset_id":"perpetration.n.01"},{"index":9,"sense_key":"commission%1:04:00::","synset_id":"mission.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"commission%1:14:01::","call_ids":["semeval2015.d002.s008.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"commission%1:14:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":761,"cached_input_tokens":0,"output_tokens":1169,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00050565,"input_uncached_usd":0.000038049999999999997,"input_cached_usd":0.0,"output_usd":0.0004676,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.445158542017452}
{"item_id":"semeval2015.d002.s008.t013","gold_sense_keys":["high%3:00:02::"],"candidates":[{"index":1,"sense_key":"high%3:00:02::","synset_id":"high.a.01"},{"index":2,"sense_key":"high%3:00:01::","synset_id":"high.a.02"},{"index":3,"sense_key":"high%5:00:00:superior:01","synset_id":"eminent.s.01"},{"index":4,"sense_key":"high%3:00:03::","synset_id":"high.a.04"},{"index":5,"sense_key":"high%5:00:00:elated:00","synset_id":"high.s.02"},{"index":6,"sense_key":"high%5:00:00:malodorous:00","synset_id":"gamey.s.02"},{"index":7,"sense_key":"high%5:00:00:intoxicated:00","synset_id":"high.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"high%3:00:02::","call_ids":["semeval2015.d002.s008.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"high%3:00:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":664,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016759999999999998,"input_uncached_usd":0.0000332,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9185228329733945}
{"item_id":"semeval2015.d002.s008.t014","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d002.s008.t014__v1__a1","semeval2015.d002.s008.t014__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":975,"cached_input_tokens":0,"output_tokens":1568,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006759499999999999,"input_uncached_usd":0.00004874999999999999,"input_cached_usd":0.0,"output_usd":0.0006272,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.324919291946571}
{"item_id":"semeval2015.d002.s008.t015","gold_sense_keys":["member%1:18:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"member%1:18:00::","call_ids":["semeval2015.d002.s008.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"member%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037104999999999994,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.85980404203292}
{"item_id":"semeval2015.d002.s008.t017","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"},{"index":2,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parliament%1:14:00::","call_ids":["semeval2015.d002.s008.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002603,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.325333666987717}
{"item_id":"semeval2015.d002.s008.t020","gold_sense_keys":["parliament%1:14:00::"],"candidates":[{"index":1,"sense_key":"parliament%1:14:00::","synset_id":"parliament.n.01"},{"index":2,"sense_key":"parliament%1:04:00::","synset_id":"fantan.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"parliament%1:14:00::","call_ids":["semeval2015.d002.s008.t020__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"parliament%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":470,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002091,"input_uncached_usd":0.0000235,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3121966660255566}
{"item_id":"semeval2015.d002.s008.t021","gold_sense_keys":["economic%3:01:01::"],"candidates":[{"index":1,"sense_key":"economic%3:01:01::","synset_id":"economic.a.01"},{"index":2,"sense_key":"economic%3:01:00::","synset_id":"economic.a.02"},{"index":3,"sense_key":"economic%5:00:00:efficient:00","synset_id":"economic.s.01"},{"index":4,"sense_key":"economic%5:00:00:worldly:00","synset_id":"economic.s.02"},{"index":5,"sense_key":"economic%5:00:00:profitable:00","synset_id":"economic.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"economic%3:01:01::","call_ids":["semeval2015.d002.s008.t021__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"economic%3:01:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":625,"cached_input_tokens":0,"output_tokens":785,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034524999999999996,"input_uncached_usd":0.00003125,"input_cached_usd":0.0,"output_usd":0.000314,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.65150924999034}
{"item_id":"semeval2015.d002.s008.t022","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s008.t022__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":637,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047344999999999993,"input_uncached_usd":0.00003185,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.382853915973101}
{"item_id":"semeval2015.d002.s008.t023","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d002.s008.t023__v1__a1","semeval2015.d002.s008.t023__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":1952,"reasoning_output_tokens":1920},"cost":{"currency":"USD","total_usd":0.00082945,"input_uncached_usd":0.000048650000000000003,"input_cached_usd":0.0,"output_usd":0.0007808,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.626442915992811}
{"item_id":"semeval2015.d002.s009.t000","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s009.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.000491,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.442622166010551}
{"item_id":"semeval2015.d002.s009.t001","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["semeval2015.d002.s009.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026964999999999997,"input_uncached_usd":0.00003285,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.670593207993079}
{"item_id":"semeval2015.d002.s009.t002","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":715,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034935,"input_uncached_usd":0.00003575,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.547836792015005}
{"item_id":"semeval2015.d002.s009.t003","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d002.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":692,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004506,"input_uncached_usd":0.0000346,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.888081250013784}
{"item_id":"semeval2015.d002.s009.t006","gold_sense_keys":["present%2:32:00::"],"candidates":[{"index":1,"sense_key":"present%2:39:00::","synset_id":"show.v.01"},{"index":2,"sense_key":"present%2:32:00::","synset_id":"present.v.02"},{"index":3,"sense_key":"present%2:36:00::","synset_id":"stage.v.01"},{"index":4,"sense_key":"present%2:40:01::","synset_id":"present.v.04"},{"index":5,"sense_key":"present%2:42:00::","synset_id":"present.v.05"},{"index":6,"sense_key":"present%2:40:02::","synset_id":"award.v.01"},{"index":7,"sense_key":"present%2:40:00::","synset_id":"give.v.08"},{"index":8,"sense_key":"present%2:32:02::","synset_id":"deliver.v.01"},{"index":9,"sense_key":"present%2:32:01::","synset_id":"introduce.v.01"},{"index":10,"sense_key":"present%2:36:08::","synset_id":"portray.v.04"},{"index":11,"sense_key":"present%2:32:06::","synset_id":"confront.v.03"},{"index":12,"sense_key":"present%2:32:04::","synset_id":"present.v.12"},{"index":13,"sense_key":"present%2:32:03::","synset_id":"salute.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"present%2:32:00::","call_ids":["semeval2015.d002.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"present%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":959,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.00092475,"input_uncached_usd":0.00004795,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.12520474998746}
{"item_id":"semeval2015.d002.s009.t007","gold_sense_keys":["policy%1:09:00::"],"candidates":[{"index":1,"sense_key":"policy%1:09:00::","synset_id":"policy.n.01"},{"index":2,"sense_key":"policy%1:10:00::","synset_id":"policy.n.02"},{"index":3,"sense_key":"policy%1:10:02::","synset_id":"policy.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"policy%1:09:00::","call_ids":["semeval2015.d002.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"policy%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":521,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021165,"input_uncached_usd":0.00002605,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.437239916995168}
{"item_id":"semeval2015.d002.s009.t008","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2015.d002.s009.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":601,"cached_input_tokens":0,"output_tokens":529,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00024165,"input_uncached_usd":0.00003005,"input_cached_usd":0.0,"output_usd":0.0002116,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.7927102080429904}
{"item_id":"semeval2015.d002.s009.t009","gold_sense_keys":["improve%2:30:01::"],"candidates":[{"index":1,"sense_key":"improve%2:30:01::","synset_id":"better.v.02"},{"index":2,"sense_key":"improve%2:30:00::","synset_id":"better.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"improve%2:30:01::","call_ids":["semeval2015.d002.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"improve%2:30:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028619999999999996,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.702550124959089}
{"item_id":"semeval2015.d002.s009.t011","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2015.d002.s009.t011__v1__a1","semeval2015.d002.s009.t011__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":973,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00047104999999999993,"input_uncached_usd":0.000048650000000000003,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.502936166070867}
{"item_id":"semeval2015.d002.s009.t012","gold_sense_keys":["development%1:04:01::"],"candidates":[{"index":1,"sense_key":"development%1:04:01::","synset_id":"development.n.01"},{"index":2,"sense_key":"development%1:22:02::","synset_id":"development.n.02"},{"index":3,"sense_key":"development%1:22:01::","synset_id":"growth.n.01"},{"index":4,"sense_key":"development%1:11:01::","synset_id":"development.n.04"},{"index":5,"sense_key":"development%1:04:02::","synset_id":"exploitation.n.01"},{"index":6,"sense_key":"development%1:15:00::","synset_id":"development.n.06"},{"index":7,"sense_key":"development%1:26:00::","synset_id":"development.n.07"},{"index":8,"sense_key":"development%1:22:00::","synset_id":"development.n.08"},{"index":9,"sense_key":"development%1:10:00::","synset_id":"development.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"development%1:04:01::","call_ids":["semeval2015.d002.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"development%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00068975,"input_uncached_usd":0.000043349999999999997,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.626644833013415}
{"item_id":"semeval2015.d002.s009.t014","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s009.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021339999999999998,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.034063500002958}
{"item_id":"semeval2015.d002.s009.t017","gold_sense_keys":["community%1:14:00::"],"candidates":[{"index":1,"sense_key":"community%1:14:00::","synset_id":"community.n.01"},{"index":2,"sense_key":"community%1:21:00::","synset_id":"community.n.02"},{"index":3,"sense_key":"community%1:14:04::","synset_id":"community.n.03"},{"index":4,"sense_key":"community%1:26:00::","synset_id":"community.n.04"},{"index":5,"sense_key":"community%1:15:00::","synset_id":"residential_district.n.01"},{"index":6,"sense_key":"community%1:14:02::","synset_id":"community.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"community%1:14:00::","call_ids":["semeval2015.d002.s009.t017__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"community%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":610,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037009999999999994,"input_uncached_usd":0.0000305,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.614952541014645}
{"item_id":"semeval2015.d002.s010.t000","gold_sense_keys":["country%1:14:00::"],"candidates":[{"index":1,"sense_key":"country%1:14:00::","synset_id":"state.n.04"},{"index":2,"sense_key":"country%1:15:00::","synset_id":"country.n.02"},{"index":3,"sense_key":"country%1:14:01::","synset_id":"nation.n.02"},{"index":4,"sense_key":"country%1:15:01::","synset_id":"country.n.04"},{"index":5,"sense_key":"country%1:15:02::","synset_id":"area.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"country%1:14:00::","call_ids":["semeval2015.d002.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"country%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":628,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047299999999999995,"input_uncached_usd":0.0000314,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.097368916962296}
{"item_id":"semeval2015.d002.s010.t005","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s010.t005__v1__a1","semeval2015.d002.s010.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":977,"cached_input_tokens":0,"output_tokens":2272,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.0009576500000000001,"input_uncached_usd":0.000048849999999999995,"input_cached_usd":0.0,"output_usd":0.0009088,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.45628508302616}
{"item_id":"semeval2015.d002.s011.t000","gold_sense_keys":["case_study%1:10:00::"],"candidates":[{"index":1,"sense_key":"case_study%1:10:00::","synset_id":"case_study.n.01"},{"index":2,"sense_key":"case_study%1:04:00::","synset_id":"case_study.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"case_study%1:10:00::","call_ids":["semeval2015.d002.s011.t000__v1__a1","semeval2015.d002.s011.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"case_study%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":937,"cached_input_tokens":0,"output_tokens":2144,"reasoning_output_tokens":2112},"cost":{"currency":"USD","total_usd":0.00090445,"input_uncached_usd":0.00004685,"input_cached_usd":0.0,"output_usd":0.0008575999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.680945417028852}
{"item_id":"semeval2015.d002.s011.t001","gold_sense_keys":["analyse%2:31:00::","analyse%2:31:04::"],"candidates":[{"index":1,"sense_key":"analyse%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"analyse%2:31:04::","synset_id":"analyze.v.03"},{"index":3,"sense_key":"analyse%2:31:01::","synset_id":"analyze.v.02"},{"index":4,"sense_key":"analyse%2:31:02::","synset_id":"analyze.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"analyse%2:31:00::","call_ids":["semeval2015.d002.s011.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"analyse%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":568,"cached_input_tokens":0,"output_tokens":657,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002912,"input_uncached_usd":0.0000284,"input_cached_usd":0.0,"output_usd":0.0002628,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.219318541989196}
{"item_id":"semeval2015.d002.s011.t002","gold_sense_keys":["background%1:09:01::","background%1:26:00::"],"candidates":[{"index":1,"sense_key":"background%1:07:00::","synset_id":"background.n.01"},{"index":2,"sense_key":"background%1:09:00::","synset_id":"background.n.02"},{"index":3,"sense_key":"background%1:09:01::","synset_id":"background.n.03"},{"index":4,"sense_key":"background%1:11:00::","synset_id":"background.n.04"},{"index":5,"sense_key":"background%1:11:01::","synset_id":"background.n.05"},{"index":6,"sense_key":"background%1:26:00::","synset_id":"setting.n.02"},{"index":7,"sense_key":"background%1:06:01::","synset_id":"background.n.07"},{"index":8,"sense_key":"background%1:06:00::","synset_id":"backdrop.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"background%1:09:01::","call_ids":["semeval2015.d002.s011.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"background%1:09:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00016969999999999998,"input_uncached_usd":0.0000353,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6595775409950875}
{"item_id":"semeval2015.d002.s011.t004","gold_sense_keys":["impact%1:19:00::"],"candidates":[{"index":1,"sense_key":"impact%1:11:00::","synset_id":"impact.n.01"},{"index":2,"sense_key":"impact%1:19:00::","synset_id":"impact.n.02"},{"index":3,"sense_key":"impact%1:04:00::","synset_id":"impingement.n.01"},{"index":4,"sense_key":"impact%1:04:01::","synset_id":"shock.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"impact%1:19:00::","call_ids":["semeval2015.d002.s011.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"impact%1:19:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":519,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031395,"input_uncached_usd":0.000025949999999999997,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.264599209011067}
{"item_id":"semeval2015.d002.s011.t005","gold_sense_keys":["employment%1:04:00::","employment%1:04:02::","employment%1:26:00::"],"candidates":[{"index":1,"sense_key":"employment%1:26:00::","synset_id":"employment.n.01"},{"index":2,"sense_key":"employment%1:04:00::","synset_id":"employment.n.02"},{"index":3,"sense_key":"employment%1:04:02::","synset_id":"employment.n.03"},{"index":4,"sense_key":"employment%1:04:01::","synset_id":"use.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"employment%1:26:00::","call_ids":["semeval2015.d002.s011.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"employment%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003658499999999999,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.0769203749950975}
{"item_id":"semeval2015.d002.s011.t007","gold_sense_keys":["work%1:04:01::"],"candidates":[{"index":1,"sense_key":"work%1:04:00::","synset_id":"work.n.01"},{"index":2,"sense_key":"work%1:06:00::","synset_id":"work.n.02"},{"index":3,"sense_key":"work%1:04:01::","synset_id":"employment.n.02"},{"index":4,"sense_key":"work%1:09:00::","synset_id":"study.n.02"},{"index":5,"sense_key":"work%1:19:00::","synset_id":"work.n.05"},{"index":6,"sense_key":"work%1:06:01::","synset_id":"workplace.n.01"},{"index":7,"sense_key":"work%1:06:02::","synset_id":"oeuvre.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"work%1:06:01::","call_ids":["semeval2015.d002.s011.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"work%1:06:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":720,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047759999999999995,"input_uncached_usd":0.000036,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.812531124975067}
{"item_id":"semeval2015.d002.s011.t009","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s011.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":539,"cached_input_tokens":0,"output_tokens":2000,"reasoning_output_tokens":1984},"cost":{"currency":"USD","total_usd":0.00082695,"input_uncached_usd":0.000026949999999999998,"input_cached_usd":0.0,"output_usd":0.0007999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.948211749957409}
{"item_id":"semeval2015.d002.s011.t010","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d002.s011.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028474999999999996,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.426755207998212}
{"item_id":"semeval2015.d002.s011.t011","gold_sense_keys":["problem%1:09:00::","problem%1:26:00::"],"candidates":[{"index":1,"sense_key":"problem%1:26:00::","synset_id":"problem.n.01"},{"index":2,"sense_key":"problem%1:10:00::","synset_id":"problem.n.02"},{"index":3,"sense_key":"problem%1:09:00::","synset_id":"trouble.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"problem%1:09:00::","call_ids":["semeval2015.d002.s011.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"problem%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023524999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.066950541047845}
{"item_id":"semeval2015.d002.s012.t000","gold_sense_keys":["visit%1:04:01::"],"candidates":[{"index":1,"sense_key":"visit%1:04:02::","synset_id":"visit.n.01"},{"index":2,"sense_key":"visit%1:14:00::","synset_id":"visit.n.02"},{"index":3,"sense_key":"visit%1:04:01::","synset_id":"visit.n.03"},{"index":4,"sense_key":"visit%1:04:00::","synset_id":"visit.n.04"},{"index":5,"sense_key":"visit%1:04:03::","synset_id":"sojourn.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"visit%1:04:02::","call_ids":["semeval2015.d002.s012.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"visit%1:04:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":535,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036595,"input_uncached_usd":0.00002675,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.0121197089902125}
{"item_id":"semeval2015.d002.s012.t001","gold_sense_keys":["interview%1:10:01::"],"candidates":[{"index":1,"sense_key":"interview%1:10:01::","synset_id":"interview.n.01"},{"index":2,"sense_key":"interview%1:10:00::","synset_id":"consultation.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"interview%1:10:01::","call_ids":["semeval2015.d002.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"interview%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":449,"cached_input_tokens":0,"output_tokens":2576,"reasoning_output_tokens":2560},"cost":{"currency":"USD","total_usd":0.00105285,"input_uncached_usd":0.00002245,"input_cached_usd":0.0,"output_usd":0.0010304,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":19.55428874999052}
{"item_id":"semeval2015.d002.s013.t000","gold_sense_keys":["national%3:00:00::","national%3:00:01::","national%3:01:00::","national%5:00:00:domestic:00"],"candidates":[{"index":1,"sense_key":"national%3:01:00::","synset_id":"national.a.01"},{"index":2,"sense_key":"national%3:00:00::","synset_id":"national.a.02"},{"index":3,"sense_key":"national%3:00:01::","synset_id":"national.a.03"},{"index":4,"sense_key":"national%5:00:00:public:00","synset_id":"national.s.01"},{"index":5,"sense_key":"national%5:00:00:domestic:00","synset_id":"home.s.01"},{"index":6,"sense_key":"national%3:01:02::","synset_id":"national.a.06"},{"index":7,"sense_key":"national%3:01:01::","synset_id":"national.a.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"national%3:00:01::","call_ids":["semeval2015.d002.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"national%3:00:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":631,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039634999999999996,"input_uncached_usd":0.00003155,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.580817749956623}
{"item_id":"semeval2015.d002.s013.t001","gold_sense_keys":["study%1:10:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d002.s013.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":779,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00063415,"input_uncached_usd":0.00003895,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.91342033300316}
{"item_id":"semeval2015.d002.s013.t002","gold_sense_keys":["form%2:42:00::"],"candidates":[{"index":1,"sense_key":"form%2:41:00::","synset_id":"form.v.01"},{"index":2,"sense_key":"form%2:42:00::","synset_id":"form.v.02"},{"index":3,"sense_key":"form%2:42:02::","synset_id":"form.v.03"},{"index":4,"sense_key":"form%2:30:01::","synset_id":"shape.v.03"},{"index":5,"sense_key":"form%2:36:00::","synset_id":"shape.v.02"},{"index":6,"sense_key":"form%2:41:02::","synset_id":"imprint.v.01"},{"index":7,"sense_key":"form%2:30:05::","synset_id":"form.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"form%2:42:00::","call_ids":["semeval2015.d002.s013.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"form%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":647,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032035,"input_uncached_usd":0.00003235,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.526514250028413}
{"item_id":"semeval2015.d002.s013.t003","gold_sense_keys":["basis%1:09:00::"],"candidates":[{"index":1,"sense_key":"basis%1:24:00::","synset_id":"footing.n.02"},{"index":2,"sense_key":"basis%1:09:00::","synset_id":"basis.n.02"},{"index":3,"sense_key":"basis%1:24:01::","synset_id":"basis.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"basis%1:09:00::","call_ids":["semeval2015.d002.s013.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"basis%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":507,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021094999999999998,"input_uncached_usd":0.00002535,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.462553833029233}
{"item_id":"semeval2015.d002.s013.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s013.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":689,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032245,"input_uncached_usd":0.00003445,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.460429500031751}
{"item_id":"semeval2015.d002.s013.t007","gold_sense_keys":["overall%5:00:00:general:00"],"candidates":[{"index":1,"sense_key":"overall%5:00:00:general:00","synset_id":"overall.s.01"},{"index":2,"sense_key":"overall%5:00:00:gross:00","synset_id":"overall.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"overall%5:00:00:general:00","call_ids":["semeval2015.d002.s013.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"overall%5:00:00:general:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":425,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033485,"input_uncached_usd":0.000021249999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.537488166010007}
{"item_id":"semeval2015.d002.s013.t008","gold_sense_keys":["practice%1:04:00::"],"candidates":[{"index":1,"sense_key":"practice%1:04:00::","synset_id":"practice.n.01"},{"index":2,"sense_key":"practice%1:04:02::","synset_id":"exercise.n.03"},{"index":3,"sense_key":"practice%1:04:04::","synset_id":"practice.n.03"},{"index":4,"sense_key":"practice%1:04:03::","synset_id":"practice.n.04"},{"index":5,"sense_key":"practice%1:09:00::","synset_id":"practice.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"practice%1:04:00::","call_ids":["semeval2015.d002.s013.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"practice%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00039349999999999997,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.122505624953192}
{"item_id":"semeval2015.d002.s014.t000","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"report%1:10:03::","call_ids":["semeval2015.d002.s014.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"report%1:10:03::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":641,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019204999999999998,"input_uncached_usd":0.00003205,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.236983207985759}
{"item_id":"semeval2015.d002.s014.t003","gold_sense_keys":["publish%2:32:00::","publish%2:36:01::"],"candidates":[{"index":1,"sense_key":"publish%2:36:01::","synset_id":"print.v.01"},{"index":2,"sense_key":"publish%2:32:00::","synset_id":"publish.v.02"},{"index":3,"sense_key":"publish%2:36:00::","synset_id":"publish.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"publish%2:32:00::","call_ids":["semeval2015.d002.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"publish%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":460,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003366,"input_uncached_usd":0.000023,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.279128999973182}
{"item_id":"semeval2015.d002.s014.t005","gold_sense_keys":["conference%1:14:00::"],"candidates":[{"index":1,"sense_key":"conference%1:14:00::","synset_id":"conference.n.01"},{"index":2,"sense_key":"conference%1:14:01::","synset_id":"league.n.01"},{"index":3,"sense_key":"conference%1:10:00::","synset_id":"conference.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"conference%1:14:00::","call_ids":["semeval2015.d002.s014.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"conference%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":418,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020649999999999998,"input_uncached_usd":0.0000209,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.642715915979352}
{"item_id":"semeval2015.d002.s014.t006","gold_sense_keys":["stage%2:36:01::"],"candidates":[{"index":1,"sense_key":"stage%2:36:00::","synset_id":"stage.v.01"},{"index":2,"sense_key":"stage%2:36:01::","synset_id":"stage.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stage%2:36:01::","call_ids":["semeval2015.d002.s014.t006__v1__a1","semeval2015.d002.s014.t006__v1__a2"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stage%2:36:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005930499999999999,"input_uncached_usd":0.00004264999999999999,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.759666458005086}
{"item_id":"semeval2015.d002.s015.t000","gold_sense_keys":["foundation%1:14:00::"],"candidates":[{"index":1,"sense_key":"foundation%1:24:00::","synset_id":"foundation.n.01"},{"index":2,"sense_key":"foundation%1:14:00::","synset_id":"foundation.n.02"},{"index":3,"sense_key":"foundation%1:06:00::","synset_id":"foundation.n.03"},{"index":4,"sense_key":"foundation%1:09:01::","synset_id":"foundation.n.04"},{"index":5,"sense_key":"foundation%1:09:00::","synset_id":"basis.n.02"},{"index":6,"sense_key":"foundation%1:06:01::","synset_id":"foundation_garment.n.01"},{"index":7,"sense_key":"foundation%1:04:00::","synset_id":"initiation.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"foundation%1:14:00::","call_ids":["semeval2015.d002.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"foundation%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":621,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026785,"input_uncached_usd":0.000031049999999999996,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.561095167009626}
{"item_id":"semeval2015.d002.s015.t001","gold_sense_keys":["aim%2:31:01::"],"candidates":[{"index":1,"sense_key":"aim%2:33:00::","synset_id":"aim.v.01"},{"index":2,"sense_key":"aim%2:31:01::","synset_id":"aim.v.02"},{"index":3,"sense_key":"aim%2:32:09::","synset_id":"drive.v.11"},{"index":4,"sense_key":"aim%2:31:02::","synset_id":"calculate.v.05"},{"index":5,"sense_key":"aim%2:33:10::","synset_id":"target.v.01"},{"index":6,"sense_key":"aim%2:32:00::","synset_id":"aim.v.06"},{"index":7,"sense_key":"aim%2:31:00::","synset_id":"draw_a_bead_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"aim%2:31:01::","call_ids":["semeval2015.d002.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"aim%2:31:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":604,"cached_input_tokens":0,"output_tokens":1617,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.0006769999999999999,"input_uncached_usd":0.0000302,"input_cached_usd":0.0,"output_usd":0.0006468,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.637884250027128}
{"item_id":"semeval2015.d002.s015.t002","gold_sense_keys":["document%2:32:00::"],"candidates":[{"index":1,"sense_key":"document%2:32:00::","synset_id":"document.v.01"},{"index":2,"sense_key":"document%2:31:00::","synset_id":"document.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%2:32:00::","call_ids":["semeval2015.d002.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":356,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.000613,"input_uncached_usd":0.0000178,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.556952207989525}
{"item_id":"semeval2015.d002.s015.t003","gold_sense_keys":["characteristic%1:07:00::","characteristic%1:09:00::"],"candidates":[{"index":1,"sense_key":"characteristic%1:09:00::","synset_id":"feature.n.01"},{"index":2,"sense_key":"characteristic%1:07:00::","synset_id":"characteristic.n.02"},{"index":3,"sense_key":"characteristic%1:10:00::","synset_id":"characteristic.n.03"},{"index":4,"sense_key":"characteristic%1:07:01::","synset_id":"characteristic.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"characteristic%1:09:00::","call_ids":["semeval2015.d002.s015.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"characteristic%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":446,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036149999999999995,"input_uncached_usd":0.0000223,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.326822250033729}
{"item_id":"semeval2015.d002.s015.t004","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s015.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":517,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049305,"input_uncached_usd":0.000025849999999999998,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.049699249968398}
{"item_id":"semeval2015.d002.s015.t005","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s015.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004363,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.122955708997324}
{"item_id":"semeval2015.d002.s015.t006","gold_sense_keys":["job%1:04:00::"],"candidates":[{"index":1,"sense_key":"job%1:04:00::","synset_id":"occupation.n.01"},{"index":2,"sense_key":"job%1:04:01::","synset_id":"job.n.02"},{"index":3,"sense_key":"job%1:06:00::","synset_id":"job.n.03"},{"index":4,"sense_key":"job%1:06:01::","synset_id":"job.n.04"},{"index":5,"sense_key":"job%1:04:04::","synset_id":"job.n.05"},{"index":6,"sense_key":"job%1:04:05::","synset_id":"job.n.06"},{"index":7,"sense_key":"job%1:04:03::","synset_id":"job.n.07"},{"index":8,"sense_key":"job%1:26:00::","synset_id":"problem.n.01"},{"index":9,"sense_key":"job%1:18:00::","synset_id":"job.n.09"},{"index":10,"sense_key":"job%1:18:01::","synset_id":"job.n.10"},{"index":11,"sense_key":"job%1:10:01::","synset_id":"job.n.11"},{"index":12,"sense_key":"job%1:10:00::","synset_id":"job.n.12"},{"index":13,"sense_key":"job%1:04:02::","synset_id":"caper.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"job%1:04:00::","call_ids":["semeval2015.d002.s015.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"job%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":845,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00038145,"input_uncached_usd":0.00004225,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.070132875000127}
{"item_id":"semeval2015.d002.s015.t008","gold_sense_keys":["training%1:04:00::"],"candidates":[{"index":1,"sense_key":"training%1:04:00::","synset_id":"training.n.01"},{"index":2,"sense_key":"training%1:07:00::","synset_id":"education.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"training%1:04:00::","call_ids":["semeval2015.d002.s015.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"training%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":369,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020404999999999997,"input_uncached_usd":0.000018449999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5610087079694495}
{"item_id":"semeval2015.d002.s015.t009","gold_sense_keys":["analyse%2:31:00::","analyse%2:31:04::"],"candidates":[{"index":1,"sense_key":"analyse%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"analyse%2:31:04::","synset_id":"analyze.v.03"},{"index":3,"sense_key":"analyse%2:31:01::","synset_id":"analyze.v.02"},{"index":4,"sense_key":"analyse%2:31:02::","synset_id":"analyze.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"analyse%2:31:00::","call_ids":["semeval2015.d002.s015.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"analyse%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":503,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013394999999999998,"input_uncached_usd":0.000025149999999999998,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.5412203339510597}
{"item_id":"semeval2015.d002.s015.t010","gold_sense_keys":["cause%1:03:00::","cause%1:11:00::"],"candidates":[{"index":1,"sense_key":"cause%1:11:00::","synset_id":"cause.n.01"},{"index":2,"sense_key":"cause%1:10:00::","synset_id":"cause.n.02"},{"index":3,"sense_key":"cause%1:04:01::","synset_id":"campaign.n.02"},{"index":4,"sense_key":"cause%1:03:00::","synset_id":"causal_agent.n.01"},{"index":5,"sense_key":"cause%1:04:00::","synset_id":"lawsuit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"cause%1:10:00::","call_ids":["semeval2015.d002.s015.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"cause%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044284999999999995,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.595324540976435}
{"item_id":"semeval2015.d002.s016.t000","gold_sense_keys":["perspective%1:09:00::"],"candidates":[{"index":1,"sense_key":"perspective%1:09:00::","synset_id":"position.n.03"},{"index":2,"sense_key":"perspective%1:07:00::","synset_id":"perspective.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perspective%1:09:00::","call_ids":["semeval2015.d002.s016.t000__v1__a1","semeval2015.d002.s016.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perspective%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":823,"cached_input_tokens":0,"output_tokens":1312,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056595,"input_uncached_usd":0.00004115,"input_cached_usd":0.0,"output_usd":0.0005248,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.292086082044989}
{"item_id":"semeval2015.d002.s016.t001","gold_sense_keys":["social%3:01:00::"],"candidates":[{"index":1,"sense_key":"social%3:01:00::","synset_id":"social.a.01"},{"index":2,"sense_key":"social%3:00:00::","synset_id":"social.a.02"},{"index":3,"sense_key":"social%3:01:02::","synset_id":"social.a.03"},{"index":4,"sense_key":"social%5:00:00:sociable:00","synset_id":"social.s.01"},{"index":5,"sense_key":"social%5:00:00:gregarious:00","synset_id":"social.s.02"},{"index":6,"sense_key":"social%5:00:00:friendly:01","synset_id":"social.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"social%3:01:00::","call_ids":["semeval2015.d002.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"social%3:01:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":556,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002646,"input_uncached_usd":0.000027799999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.730993791949004}
{"item_id":"semeval2015.d002.s016.t004","gold_sense_keys":["particular%5:00:00:uncommon:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":3,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":5,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"particular%5:00:00:uncommon:00","call_ids":["semeval2015.d002.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"particular%5:00:00:uncommon:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":606,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005743,"input_uncached_usd":0.000030299999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.569008207996376}
{"item_id":"semeval2015.d002.s016.t005","gold_sense_keys":["attention%1:09:00::"],"candidates":[{"index":1,"sense_key":"attention%1:09:00::","synset_id":"attention.n.01"},{"index":2,"sense_key":"attention%1:04:01::","synset_id":"care.n.01"},{"index":3,"sense_key":"attention%1:09:02::","synset_id":"attention.n.03"},{"index":4,"sense_key":"attention%1:04:02::","synset_id":"attention.n.04"},{"index":5,"sense_key":"attention%1:09:01::","synset_id":"attention.n.05"},{"index":6,"sense_key":"attention%1:07:00::","synset_id":"attention.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"attention%1:09:00::","call_ids":["semeval2015.d002.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"attention%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":552,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006228000000000001,"input_uncached_usd":0.0000276,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.82539341703523}
{"item_id":"semeval2015.d002.s016.t006","gold_sense_keys":["pay%2:31:00::","pay%2:32:00::"],"candidates":[{"index":1,"sense_key":"pay%2:40:00::","synset_id":"pay.v.01"},{"index":2,"sense_key":"pay%2:32:00::","synset_id":"give.v.05"},{"index":3,"sense_key":"pay%2:40:04::","synset_id":"pay_up.v.01"},{"index":4,"sense_key":"pay%2:40:01::","synset_id":"yield.v.10"},{"index":5,"sense_key":"pay%2:40:02::","synset_id":"pay.v.05"},{"index":6,"sense_key":"pay%2:31:00::","synset_id":"give.v.10"},{"index":7,"sense_key":"pay%2:42:00::","synset_id":"pay.v.07"},{"index":8,"sense_key":"pay%2:41:00::","synset_id":"pay.v.08"},{"index":9,"sense_key":"pay%2:31:03::","synset_id":"pay.v.09"},{"index":10,"sense_key":"pay%2:40:13::","synset_id":"pay.v.10"},{"index":11,"sense_key":"pay%2:40:14::","synset_id":"pay.v.11"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"pay%2:31:00::","call_ids":["semeval2015.d002.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"pay%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022189999999999997,"input_uncached_usd":0.0000363,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.454492249991745}
{"item_id":"semeval2015.d002.s016.t007","gold_sense_keys":["proposal%1:10:00::"],"candidates":[{"index":1,"sense_key":"proposal%1:10:00::","synset_id":"proposal.n.01"},{"index":2,"sense_key":"proposal%1:10:02::","synset_id":"marriage_proposal.n.01"},{"index":3,"sense_key":"proposal%1:10:01::","synset_id":"proposal.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"proposal%1:10:00::","call_ids":["semeval2015.d002.s016.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"proposal%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":402,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020569999999999999,"input_uncached_usd":0.0000201,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8749757079640403}
{"item_id":"semeval2015.d002.s016.t008","gold_sense_keys":["remove%2:30:00::","remove%2:40:00::"],"candidates":[{"index":1,"sense_key":"remove%2:30:00::","synset_id":"remove.v.01"},{"index":2,"sense_key":"remove%2:41:00::","synset_id":"remove.v.02"},{"index":3,"sense_key":"remove%2:40:00::","synset_id":"get_rid_of.v.01"},{"index":4,"sense_key":"remove%2:41:02::","synset_id":"take_out.v.01"},{"index":5,"sense_key":"remove%2:38:00::","synset_id":"remove.v.05"},{"index":6,"sense_key":"remove%2:30:01::","synset_id":"absent.v.01"},{"index":7,"sense_key":"remove%2:41:01::","synset_id":"murder.v.01"},{"index":8,"sense_key":"remove%2:30:02::","synset_id":"remove.v.08"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"remove%2:30:02::","call_ids":["semeval2015.d002.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"remove%2:30:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":657,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044885,"input_uncached_usd":0.00003285,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.174177624983713}
{"item_id":"semeval2015.d002.s016.t009","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"age%1:28:01::","call_ids":["semeval2015.d002.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"age%1:28:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":526,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0005959000000000001,"input_uncached_usd":0.0000263,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.13352312502684}
{"item_id":"semeval2015.d002.s016.t010","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s016.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":406,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004363,"input_uncached_usd":0.0000203,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.74174425000092}
{"item_id":"semeval2015.d002.s017.t002","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d002.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":483,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026095,"input_uncached_usd":0.000024149999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.653159832989331}
{"item_id":"semeval2015.d002.s017.t005","gold_sense_keys":["private%3:00:00::"],"candidates":[{"index":1,"sense_key":"private%3:00:00::","synset_id":"private.a.01"},{"index":2,"sense_key":"private%5:00:02:personal:00","synset_id":"private.s.01"},{"index":3,"sense_key":"private%5:00:00:personal:00","synset_id":"individual.s.03"},{"index":4,"sense_key":"private%5:00:00:inward:00","synset_id":"secret.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"private%3:00:00::","call_ids":["semeval2015.d002.s017.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"private%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":454,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046429999999999996,"input_uncached_usd":0.0000227,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.096217958023772}
{"item_id":"semeval2015.d002.s017.t006","gold_sense_keys":["sector%1:14:00::"],"candidates":[{"index":1,"sense_key":"sector%1:25:00::","synset_id":"sector.n.01"},{"index":2,"sense_key":"sector%1:14:00::","synset_id":"sector.n.02"},{"index":3,"sense_key":"sector%1:09:00::","synset_id":"sector.n.03"},{"index":4,"sense_key":"sector%1:23:00::","synset_id":"sector.n.04"},{"index":5,"sense_key":"sector%1:15:00::","synset_id":"sector.n.05"},{"index":6,"sense_key":"sector%1:06:00::","synset_id":"sector.n.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"sector%1:14:00::","call_ids":["semeval2015.d002.s017.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"sector%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015945,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.8559012499754317}
{"item_id":"semeval2015.d002.s017.t007","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s017.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":524,"cached_input_tokens":0,"output_tokens":1808,"reasoning_output_tokens":1792},"cost":{"currency":"USD","total_usd":0.0007494,"input_uncached_usd":0.0000262,"input_cached_usd":0.0,"output_usd":0.0007232,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.496361166995484}
{"item_id":"semeval2015.d002.s017.t008","gold_sense_keys":["barrier%1:09:00::"],"candidates":[{"index":1,"sense_key":"barrier%1:06:00::","synset_id":"barrier.n.01"},{"index":2,"sense_key":"barrier%1:09:00::","synset_id":"barrier.n.02"},{"index":3,"sense_key":"barrier%1:17:00::","synset_id":"barrier.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"barrier%1:09:00::","call_ids":["semeval2015.d002.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"barrier%1:09:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":404,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002826,"input_uncached_usd":0.0000202,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.930074749980122}
{"item_id":"semeval2015.d002.s017.t009","gold_sense_keys":["especially%4:02:00::"],"candidates":[{"index":1,"sense_key":"especially%4:02:00::","synset_id":"particularly.r.01"},{"index":2,"sense_key":"especially%4:02:01::","synset_id":"specially.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"especially%4:02:00::","call_ids":["semeval2015.d002.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"especially%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":386,"cached_input_tokens":0,"output_tokens":1041,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043569999999999997,"input_uncached_usd":0.000019299999999999998,"input_cached_usd":0.0,"output_usd":0.0004164,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.194931540987454}
{"item_id":"semeval2015.d002.s017.t010","gold_sense_keys":["older%5:00:00:experienced:00","older%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"older%5:00:00:old:02","synset_id":"aged.s.01"},{"index":2,"sense_key":"older%5:00:00:senior:00","synset_id":"elder.s.01"},{"index":3,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":4,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":5,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":6,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":7,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":8,"sense_key":"erstwhile%5:00:00:past:00","synset_id":"erstwhile.s.01"},{"index":9,"sense_key":"honest-to-god%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":10,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":11,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"older%5:00:00:old:02","call_ids":["semeval2015.d002.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"older%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":860,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006126,"input_uncached_usd":0.000042999999999999995,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.572191958024632}
{"item_id":"semeval2015.d002.s017.t011","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":468,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020899999999999998,"input_uncached_usd":0.0000234,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.608502042014152}
{"item_id":"semeval2015.d002.s018.t000","gold_sense_keys":["seek%2:35:00::"],"candidates":[{"index":1,"sense_key":"seek%2:40:00::","synset_id":"seek.v.01"},{"index":2,"sense_key":"seek%2:35:00::","synset_id":"search.v.01"},{"index":3,"sense_key":"seek%2:41:00::","synset_id":"try.v.01"},{"index":4,"sense_key":"seek%2:38:00::","synset_id":"seek.v.04"},{"index":5,"sense_key":"seek%2:32:00::","synset_id":"seek.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"seek%2:35:00::","call_ids":["semeval2015.d002.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"seek%2:35:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003391,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.454961624986026}
{"item_id":"semeval2015.d002.s018.t002","gold_sense_keys":["early%3:00:00::"],"candidates":[{"index":1,"sense_key":"early%3:00:00::","synset_id":"early.a.01"},{"index":2,"sense_key":"early%3:00:02::","synset_id":"early.a.02"},{"index":3,"sense_key":"early%5:00:00:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"early%5:00:00:young:00","synset_id":"early.s.02"},{"index":5,"sense_key":"early%3:00:01::","synset_id":"early.a.05"},{"index":6,"sense_key":"early%5:00:01:future:00","synset_id":"early.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"early%3:00:00::","call_ids":["semeval2015.d002.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"early%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":574,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003167,"input_uncached_usd":0.0000287,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.095983500010334}
{"item_id":"semeval2015.d002.s018.t003","gold_sense_keys":["retirement%1:04:00::","retirement%1:26:00::"],"candidates":[{"index":1,"sense_key":"retirement%1:26:00::","synset_id":"retirement.n.01"},{"index":2,"sense_key":"retirement%1:04:00::","synset_id":"retirement.n.02"},{"index":3,"sense_key":"retirement%1:04:01::","synset_id":"retirement.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"retirement%1:04:00::","call_ids":["semeval2015.d002.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"retirement%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.0004619499999999999,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.035172584000975}
{"item_id":"semeval2015.d002.s018.t004","gold_sense_keys":["redundancy%1:07:00::"],"candidates":[{"index":1,"sense_key":"redundancy%1:10:00::","synset_id":"redundancy.n.01"},{"index":2,"sense_key":"redundancy%1:07:00::","synset_id":"redundancy.n.02"},{"index":3,"sense_key":"redundancy%1:09:00::","synset_id":"redundancy.n.03"},{"index":4,"sense_key":"redundancy%1:04:00::","synset_id":"redundancy.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"redundancy%1:07:00::","call_ids":["semeval2015.d002.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"redundancy%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025919999999999996,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.15622008300852}
{"item_id":"semeval2015.d002.s018.t007","gold_sense_keys":["concerned%5:00:00:involved:00"],"candidates":[{"index":1,"sense_key":"concerned%3:00:00::","synset_id":"concerned.a.01"},{"index":2,"sense_key":"concerned%5:00:00:involved:00","synset_id":"concerned.s.01"},{"index":3,"sense_key":"concerned%5:00:01:involved:00","synset_id":"implicated.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"concerned%5:00:00:involved:00","call_ids":["semeval2015.d002.s018.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"concerned%5:00:00:involved:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":466,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031130000000000003,"input_uncached_usd":0.0000233,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.9756669999915175}
{"item_id":"semeval2015.d002.s018.t008","gold_sense_keys":["retention%1:04:00::"],"candidates":[{"index":1,"sense_key":"retention%1:04:00::","synset_id":"retention.n.01"},{"index":2,"sense_key":"retention%1:09:00::","synset_id":"memory.n.03"},{"index":3,"sense_key":"retention%1:07:00::","synset_id":"retentiveness.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"retention%1:04:00::","call_ids":["semeval2015.d002.s018.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"retention%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":433,"cached_input_tokens":0,"output_tokens":144,"reasoning_output_tokens":128},"cost":{"currency":"USD","total_usd":0.00007925,"input_uncached_usd":0.000021649999999999998,"input_cached_usd":0.0,"output_usd":0.0000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.9583492920501158}
{"item_id":"semeval2015.d002.s018.t011","gold_sense_keys":["older%5:00:00:experienced:00","older%5:00:00:old:02"],"candidates":[{"index":1,"sense_key":"older%5:00:00:old:02","synset_id":"aged.s.01"},{"index":2,"sense_key":"older%5:00:00:senior:00","synset_id":"elder.s.01"},{"index":3,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":4,"sense_key":"old%3:00:02::","synset_id":"old.a.01"},{"index":5,"sense_key":"old%3:00:01::","synset_id":"old.a.02"},{"index":6,"sense_key":"old%5:00:00:familiar:00","synset_id":"old.s.01"},{"index":7,"sense_key":"older%5:00:00:experienced:00","synset_id":"old.s.02"},{"index":8,"sense_key":"erstwhile%5:00:00:past:00","synset_id":"erstwhile.s.01"},{"index":9,"sense_key":"honest-to-god%5:00:00:genuine:00","synset_id":"honest-to-god.s.01"},{"index":10,"sense_key":"old%5:00:00:early:01","synset_id":"old.s.05"},{"index":11,"sense_key":"previous%5:00:00:preceding:00","synset_id":"previous.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"older%5:00:00:old:02","call_ids":["semeval2015.d002.s018.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"older%5:00:00:old:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":867,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00056175,"input_uncached_usd":0.000043349999999999997,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.960533124976791}
{"item_id":"semeval2015.d002.s018.t012","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s018.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":475,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038855,"input_uncached_usd":0.000023749999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.932675249991007}
{"item_id":"semeval2015.d002.s019.t000","gold_sense_keys":["gender%1:07:00::"],"candidates":[{"index":1,"sense_key":"gender%1:10:00::","synset_id":"gender.n.01"},{"index":2,"sense_key":"gender%1:07:00::","synset_id":"sex.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"gender%1:07:00::","call_ids":["semeval2015.d002.s019.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"gender%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.0005396,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.195236540981568}
{"item_id":"semeval2015.d002.s019.t001","gold_sense_keys":["ethnic%5:00:00:social:00"],"candidates":[{"index":1,"sense_key":"ethnic%5:00:00:social:00","synset_id":"cultural.s.01"},{"index":2,"sense_key":"ethnic%5:00:00:irreligious:00","synset_id":"heathen.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"ethnic%5:00:00:social:00","call_ids":["semeval2015.d002.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"ethnic%5:00:00:social:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":337,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.0001561,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0001348,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3848100840114057}
{"item_id":"semeval2015.d002.s019.t002","gold_sense_keys":["background%1:07:00::"],"candidates":[{"index":1,"sense_key":"background%1:07:00::","synset_id":"background.n.01"},{"index":2,"sense_key":"background%1:09:00::","synset_id":"background.n.02"},{"index":3,"sense_key":"background%1:09:01::","synset_id":"background.n.03"},{"index":4,"sense_key":"background%1:11:00::","synset_id":"background.n.04"},{"index":5,"sense_key":"background%1:11:01::","synset_id":"background.n.05"},{"index":6,"sense_key":"background%1:26:00::","synset_id":"setting.n.02"},{"index":7,"sense_key":"background%1:06:01::","synset_id":"background.n.07"},{"index":8,"sense_key":"background%1:06:00::","synset_id":"backdrop.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"background%1:07:00::","call_ids":["semeval2015.d002.s019.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"background%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00029545,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.867531833006069}
{"item_id":"semeval2015.d002.s019.t003","gold_sense_keys":["worker%1:18:00::"],"candidates":[{"index":1,"sense_key":"worker%1:18:00::","synset_id":"worker.n.01"},{"index":2,"sense_key":"worker%1:18:01::","synset_id":"proletarian.n.01"},{"index":3,"sense_key":"worker%1:05:01::","synset_id":"worker.n.03"},{"index":4,"sense_key":"worker%1:18:02::","synset_id":"actor.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"worker%1:18:00::","call_ids":["semeval2015.d002.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"worker%1:18:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":479,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020954999999999997,"input_uncached_usd":0.00002395,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.321244333055802}
{"item_id":"semeval2015.d002.s019.t004","gold_sense_keys":["relate%2:42:00::","relate%2:42:01::"],"candidates":[{"index":1,"sense_key":"relate%2:31:00::","synset_id":"associate.v.01"},{"index":2,"sense_key":"relate%2:42:00::","synset_id":"refer.v.02"},{"index":3,"sense_key":"relate%2:32:00::","synset_id":"relate.v.03"},{"index":4,"sense_key":"relate%2:42:01::","synset_id":"relate.v.04"},{"index":5,"sense_key":"relate%2:41:00::","synset_id":"relate.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"relate%2:42:00::","call_ids":["semeval2015.d002.s019.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"relate%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":536,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004172,"input_uncached_usd":0.000026799999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.34365583298495}
{"item_id":"semeval2015.d002.s019.t005","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":534,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018669999999999998,"input_uncached_usd":0.0000267,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.527817042020615}
{"item_id":"semeval2015.d002.s019.t006","gold_sense_keys":["discrimination%1:04:00::"],"candidates":[{"index":1,"sense_key":"discrimination%1:04:00::","synset_id":"discrimination.n.01"},{"index":2,"sense_key":"discrimination%1:09:00::","synset_id":"discrimination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discrimination%1:04:00::","call_ids":["semeval2015.d002.s019.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discrimination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":389,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00056345,"input_uncached_usd":0.00001945,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.138165250013117}
{"item_id":"semeval2015.d002.s019.t007","gold_sense_keys":["examine%2:31:00::"],"candidates":[{"index":1,"sense_key":"examine%2:31:00::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"examine%2:39:00::","synset_id":"examine.v.02"},{"index":3,"sense_key":"examine%2:32:01::","synset_id":"probe.v.01"},{"index":4,"sense_key":"examine%2:32:00::","synset_id":"examine.v.04"},{"index":5,"sense_key":"examine%2:41:00::","synset_id":"test.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"examine%2:31:00::","call_ids":["semeval2015.d002.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"examine%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":541,"cached_input_tokens":0,"output_tokens":593,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026425,"input_uncached_usd":0.00002705,"input_cached_usd":0.0,"output_usd":0.0002372,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.253629250044469}
{"item_id":"semeval2015.d002.s020.t002","gold_sense_keys":["consider%2:39:00::"],"candidates":[{"index":1,"sense_key":"consider%2:31:00::","synset_id":"see.v.05"},{"index":2,"sense_key":"consider%2:39:00::","synset_id":"study.v.03"},{"index":3,"sense_key":"consider%2:31:01::","synset_id":"consider.v.03"},{"index":4,"sense_key":"consider%2:32:00::","synset_id":"consider.v.04"},{"index":5,"sense_key":"consider%2:32:02::","synset_id":"consider.v.05"},{"index":6,"sense_key":"consider%2:31:08::","synset_id":"think.v.01"},{"index":7,"sense_key":"consider%2:39:04::","synset_id":"regard.v.02"},{"index":8,"sense_key":"consider%2:39:08::","synset_id":"view.v.02"},{"index":9,"sense_key":"consider%2:31:05::","synset_id":"consider.v.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"consider%2:32:02::","call_ids":["semeval2015.d002.s020.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"consider%2:32:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":681,"cached_input_tokens":0,"output_tokens":849,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037364999999999995,"input_uncached_usd":0.00003405,"input_cached_usd":0.0,"output_usd":0.00033959999999999996,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.267961415986065}
{"item_id":"semeval2015.d002.s020.t004","gold_sense_keys":["age%1:07:00::"],"candidates":[{"index":1,"sense_key":"age%1:07:00::","synset_id":"age.n.01"},{"index":2,"sense_key":"age%1:28:02::","synset_id":"historic_period.n.01"},{"index":3,"sense_key":"age%1:28:00::","synset_id":"age.n.03"},{"index":4,"sense_key":"age%1:28:03::","synset_id":"long_time.n.01"},{"index":5,"sense_key":"age%1:28:01::","synset_id":"old_age.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"age%1:28:00::","call_ids":["semeval2015.d002.s020.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"age%1:28:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":528,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004168,"input_uncached_usd":0.000026399999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.052403333014809}
{"item_id":"semeval2015.d002.s020.t005","gold_sense_keys":["discrimination%1:04:00::"],"candidates":[{"index":1,"sense_key":"discrimination%1:04:00::","synset_id":"discrimination.n.01"},{"index":2,"sense_key":"discrimination%1:09:00::","synset_id":"discrimination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"discrimination%1:04:00::","call_ids":["semeval2015.d002.s020.t005__v1__a1","semeval2015.d002.s020.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"discrimination%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":813,"cached_input_tokens":0,"output_tokens":928,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00041185,"input_uncached_usd":0.00004065,"input_cached_usd":0.0,"output_usd":0.0003712,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.467004291946068}
{"item_id":"semeval2015.d002.s020.t007","gold_sense_keys":["small%5:00:00:limited:00"],"candidates":[{"index":1,"sense_key":"small%3:00:00::","synset_id":"small.a.01"},{"index":2,"sense_key":"small%5:00:00:limited:00","synset_id":"minor.s.03"},{"index":3,"sense_key":"small%5:00:00:young:00","synset_id":"little.s.01"},{"index":4,"sense_key":"small%5:00:00:little:03","synset_id":"small.s.03"},{"index":5,"sense_key":"small%5:00:00:inferior:01","synset_id":"humble.s.01"},{"index":6,"sense_key":"small%5:00:00:lowercase:00","synset_id":"little.s.04"},{"index":7,"sense_key":"small%5:00:00:soft:04","synset_id":"little.s.03"},{"index":8,"sense_key":"small%5:00:00:fine:00","synset_id":"small.s.07"},{"index":9,"sense_key":"small%5:00:00:moderate:00","synset_id":"modest.s.01"},{"index":10,"sense_key":"small%5:00:00:decreased:00","synset_id":"belittled.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"small%5:00:00:limited:00","call_ids":["semeval2015.d002.s020.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"small%5:00:00:limited:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":745,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00047884999999999996,"input_uncached_usd":0.00003725,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.939501624961849}
{"item_id":"semeval2015.d002.s020.t008","gold_sense_keys":["enterprise%1:14:00::"],"candidates":[{"index":1,"sense_key":"enterprise%1:04:00::","synset_id":"enterprise.n.01"},{"index":2,"sense_key":"enterprise%1:14:00::","synset_id":"enterprise.n.02"},{"index":3,"sense_key":"enterprise%1:07:00::","synset_id":"enterprise.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"enterprise%1:14:00::","call_ids":["semeval2015.d002.s020.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"enterprise%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038609999999999995,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.629683250037488}
{"item_id":"semeval2015.d002.s021.t000","gold_sense_keys":["member%1:14:00::"],"candidates":[{"index":1,"sense_key":"member%1:18:00::","synset_id":"member.n.01"},{"index":2,"sense_key":"member%1:24:00::","synset_id":"member.n.02"},{"index":3,"sense_key":"member%1:08:00::","synset_id":"extremity.n.01"},{"index":4,"sense_key":"member%1:14:00::","synset_id":"member.n.04"},{"index":5,"sense_key":"member%1:08:01::","synset_id":"penis.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"member%1:14:00::","call_ids":["semeval2015.d002.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"member%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":537,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018685,"input_uncached_usd":0.00002685,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.487321541993879}
{"item_id":"semeval2015.d002.s021.t002","gold_sense_keys":["cover%2:42:02::"],"candidates":[{"index":1,"sense_key":"cover%2:35:00::","synset_id":"cover.v.01"},{"index":2,"sense_key":"cover%2:35:01::","synset_id":"cover.v.02"},{"index":3,"sense_key":"cover%2:42:01::","synset_id":"cover.v.03"},{"index":4,"sense_key":"cover%2:42:00::","synset_id":"cover.v.04"},{"index":5,"sense_key":"cover%2:32:01::","synset_id":"cover.v.05"},{"index":6,"sense_key":"cover%2:42:02::","synset_id":"embrace.v.01"},{"index":7,"sense_key":"cover%2:38:00::","synset_id":"traverse.v.01"},{"index":8,"sense_key":"cover%2:32:02::","synset_id":"report.v.05"},{"index":9,"sense_key":"cover%2:33:01::","synset_id":"cover.v.09"},{"index":10,"sense_key":"cover%2:41:01::","synset_id":"cover.v.10"},{"index":11,"sense_key":"cover%2:39:12::","synset_id":"cover.v.11"},{"index":12,"sense_key":"cover%2:33:00::","synset_id":"cover.v.12"},{"index":13,"sense_key":"cover%2:32:12::","synset_id":"cover.v.13"},{"index":14,"sense_key":"cover%2:32:00::","synset_id":"cover.v.14"},{"index":15,"sense_key":"cover%2:42:14::","synset_id":"cover.v.15"},{"index":16,"sense_key":"cover%2:41:08::","synset_id":"cover.v.16"},{"index":17,"sense_key":"cover%2:41:10::","synset_id":"cover.v.17"},{"index":18,"sense_key":"cover%2:40:00::","synset_id":"cover.v.18"},{"index":19,"sense_key":"cover%2:39:14::","synset_id":"cover.v.19"},{"index":20,"sense_key":"cover%2:35:13::","synset_id":"shroud.v.01"},{"index":21,"sense_key":"cover%2:35:02::","synset_id":"breed.v.02"},{"index":22,"sense_key":"cover%2:35:14::","synset_id":"overlay.v.01"},{"index":23,"sense_key":"cover%2:33:07::","synset_id":"cover.v.23"},{"index":24,"sense_key":"cover%2:33:13::","synset_id":"cover.v.24"},{"index":25,"sense_key":"cover%2:29:02::","synset_id":"brood.v.05"},{"index":26,"sense_key":"cover%2:29:15::","synset_id":"cover.v.26"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"cover%2:42:02::","call_ids":["semeval2015.d002.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"cover%2:42:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1356,"cached_input_tokens":0,"output_tokens":1489,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006634,"input_uncached_usd":0.0000678,"input_cached_usd":0.0,"output_usd":0.0005956,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.624921832990367}
{"item_id":"semeval2015.d003.s000.t000","gold_sense_keys":["document%1:10:00::"],"candidates":[{"index":1,"sense_key":"document%1:10:00::","synset_id":"document.n.01"},{"index":2,"sense_key":"document%1:06:00::","synset_id":"document.n.02"},{"index":3,"sense_key":"document%1:21:00::","synset_id":"document.n.03"},{"index":4,"sense_key":"document%1:10:01::","synset_id":"text_file.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"document%1:10:00::","call_ids":["semeval2015.d003.s000.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"document%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":319,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043194999999999996,"input_uncached_usd":0.000015949999999999998,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.669170542038046}
{"item_id":"semeval2015.d003.s000.t003","gold_sense_keys":["public%3:00:00::"],"candidates":[{"index":1,"sense_key":"public%3:00:00::","synset_id":"public.a.01"},{"index":2,"sense_key":"public%5:00:00:common:02","synset_id":"public.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"public%3:00:00::","call_ids":["semeval2015.d003.s000.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"public%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":268,"cached_input_tokens":0,"output_tokens":2320,"reasoning_output_tokens":2304},"cost":{"currency":"USD","total_usd":0.0009414,"input_uncached_usd":0.000013399999999999999,"input_cached_usd":0.0,"output_usd":0.000928,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.160688959003892}
{"item_id":"semeval2015.d003.s000.t004","gold_sense_keys":["assessment%1:04:00::"],"candidates":[{"index":1,"sense_key":"assessment%1:09:00::","synset_id":"appraisal.n.01"},{"index":2,"sense_key":"assessment%1:21:00::","synset_id":"assessment.n.02"},{"index":3,"sense_key":"assessment%1:07:00::","synset_id":"assessment.n.03"},{"index":4,"sense_key":"assessment%1:04:00::","synset_id":"judgment.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"assessment%1:04:00::","call_ids":["semeval2015.d003.s000.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"assessment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":329,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038124999999999997,"input_uncached_usd":0.00001645,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.6749214170267805}
{"item_id":"semeval2015.d003.s000.t005","gold_sense_keys":["report%1:10:03::"],"candidates":[{"index":1,"sense_key":"report%1:10:03::","synset_id":"report.n.01"},{"index":2,"sense_key":"report%1:10:01::","synset_id":"report.n.02"},{"index":3,"sense_key":"report%1:10:00::","synset_id":"report.n.03"},{"index":4,"sense_key":"report%1:11:00::","synset_id":"report.n.04"},{"index":5,"sense_key":"report%1:10:04::","synset_id":"report_card.n.01"},{"index":6,"sense_key":"report%1:10:02::","synset_id":"composition.n.08"},{"index":7,"sense_key":"report%1:09:00::","synset_id":"reputation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"report%1:10:00::","call_ids":["semeval2015.d003.s000.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"report%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":516,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003138,"input_uncached_usd":0.0000258,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.776140416972339}
{"item_id":"semeval2015.d003.s001.t000","gold_sense_keys":["explain%2:32:00::","explain%2:32:01::"],"candidates":[{"index":1,"sense_key":"explain%2:32:00::","synset_id":"explain.v.01"},{"index":2,"sense_key":"explain%2:32:01::","synset_id":"explain.v.02"},{"index":3,"sense_key":"explain%2:32:02::","synset_id":"excuse.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"explain%2:32:00::","call_ids":["semeval2015.d003.s001.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"explain%2:32:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017664999999999998,"input_uncached_usd":0.000016649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.156868874968495}
{"item_id":"semeval2015.d003.s001.t001","gold_sense_keys":["committee%1:14:00::"],"candidates":[{"index":1,"sense_key":"committee%1:14:00::","synset_id":"committee.n.01"},{"index":2,"sense_key":"committee%1:14:01::","synset_id":"committee.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"committee%1:14:00::","call_ids":["semeval2015.d003.s001.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"committee%1:14:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":276,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004042,"input_uncached_usd":0.0000138,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.483805708005093}
{"item_id":"semeval2015.d003.s001.t003","gold_sense_keys":["product%1:06:01::"],"candidates":[{"index":1,"sense_key":"product%1:06:01::","synset_id":"merchandise.n.01"},{"index":2,"sense_key":"product%1:06:00::","synset_id":"product.n.02"},{"index":3,"sense_key":"product%1:09:00::","synset_id":"product.n.03"},{"index":4,"sense_key":"product%1:27:00::","synset_id":"product.n.04"},{"index":5,"sense_key":"product%1:19:00::","synset_id":"product.n.05"},{"index":6,"sense_key":"product%1:14:00::","synset_id":"intersection.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"product%1:06:00::","call_ids":["semeval2015.d003.s001.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"product%1:06:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":478,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005167,"input_uncached_usd":0.000023899999999999998,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.733014625031501}
{"item_id":"semeval2015.d003.s001.t005","gold_sense_keys":["use%1:04:00::"],"candidates":[{"index":1,"sense_key":"use%1:04:00::","synset_id":"use.n.01"},{"index":2,"sense_key":"use%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"use%1:07:02::","synset_id":"use.n.03"},{"index":4,"sense_key":"use%1:22:00::","synset_id":"consumption.n.03"},{"index":5,"sense_key":"use%1:04:01::","synset_id":"habit.n.02"},{"index":6,"sense_key":"use%1:04:02::","synset_id":"manipulation.n.01"},{"index":7,"sense_key":"use%1:07:03::","synset_id":"use.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"use%1:07:00::","call_ids":["semeval2015.d003.s001.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"use%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":555,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.00062295,"input_uncached_usd":0.00002775,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.885268291982356}
{"item_id":"semeval2015.d003.s001.t006","gold_sense_keys":["assess%2:31:00::"],"candidates":[{"index":1,"sense_key":"assess%2:31:00::","synset_id":"measure.v.04"},{"index":2,"sense_key":"assess%2:40:01::","synset_id":"assess.v.02"},{"index":3,"sense_key":"assess%2:40:00::","synset_id":"tax.v.02"},{"index":4,"sense_key":"assess%2:31:01::","synset_id":"assess.v.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"assess%2:31:00::","call_ids":["semeval2015.d003.s001.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"assess%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":378,"cached_input_tokens":0,"output_tokens":208,"reasoning_output_tokens":192},"cost":{"currency":"USD","total_usd":0.00010209999999999999,"input_uncached_usd":0.0000189,"input_cached_usd":0.0,"output_usd":0.00008319999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":1.7020585000282153}
{"item_id":"semeval2015.d003.s001.t007","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%1:04:00::","call_ids":["semeval2015.d003.s001.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":613,"cached_input_tokens":0,"output_tokens":2833,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.00116385,"input_uncached_usd":0.00003065,"input_cached_usd":0.0,"output_usd":0.0011332,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.15337241702946}
{"item_id":"semeval2015.d003.s001.t008","gold_sense_keys":["perform%2:36:00::"],"candidates":[{"index":1,"sense_key":"perform%2:36:00::","synset_id":"perform.v.01"},{"index":2,"sense_key":"perform%2:41:02::","synset_id":"perform.v.02"},{"index":3,"sense_key":"perform%2:36:01::","synset_id":"perform.v.03"},{"index":4,"sense_key":"perform%2:41:00::","synset_id":"do.v.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"perform%2:36:00::","call_ids":["semeval2015.d003.s001.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"perform%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":358,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001779,"input_uncached_usd":0.000017899999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.2652056249789894}
{"item_id":"semeval2015.d003.s001.t009","gold_sense_keys":["reach%2:38:01::","reach%2:38:06::","reach%2:41:00::"],"candidates":[{"index":1,"sense_key":"reach%2:38:01::","synset_id":"reach.v.01"},{"index":2,"sense_key":"reach%2:38:00::","synset_id":"reach.v.02"},{"index":3,"sense_key":"reach%2:35:00::","synset_id":"reach.v.03"},{"index":4,"sense_key":"reach%2:32:00::","synset_id":"reach.v.04"},{"index":5,"sense_key":"reach%2:41:00::","synset_id":"achieve.v.01"},{"index":6,"sense_key":"reach%2:42:00::","synset_id":"reach.v.06"},{"index":7,"sense_key":"reach%2:38:06::","synset_id":"reach.v.07"},{"index":8,"sense_key":"reach%2:40:00::","synset_id":"pass.v.05"},{"index":9,"sense_key":"reach%2:33:00::","synset_id":"strive.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"reach%2:38:06::","call_ids":["semeval2015.d003.s001.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"reach%2:38:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":1876,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007815499999999999,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.0007503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.989158125012182}
{"item_id":"semeval2015.d003.s001.t010","gold_sense_keys":["recommendation%1:10:00::"],"candidates":[{"index":1,"sense_key":"recommendation%1:10:00::","synset_id":"recommendation.n.01"},{"index":2,"sense_key":"recommendation%1:10:02::","synset_id":"recommendation.n.02"},{"index":3,"sense_key":"recommendation%1:07:00::","synset_id":"recommendation.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"recommendation%1:10:00::","call_ids":["semeval2015.d003.s001.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"recommendation%1:10:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":333,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00017664999999999998,"input_uncached_usd":0.000016649999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.1100051249959506}
{"item_id":"semeval2015.d003.s001.t011","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s001.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":480,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028639999999999997,"input_uncached_usd":0.000024,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.096672834013589}
{"item_id":"semeval2015.d003.s001.t012","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s001.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003065,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.023184416000731}
{"item_id":"semeval2015.d003.s003.t000","gold_sense_keys":["contain%2:42:00::"],"candidates":[{"index":1,"sense_key":"contain%2:42:00::","synset_id":"incorporate.v.02"},{"index":2,"sense_key":"contain%2:42:13::","synset_id":"hold.v.11"},{"index":3,"sense_key":"contain%2:41:00::","synset_id":"control.v.02"},{"index":4,"sense_key":"contain%2:42:06::","synset_id":"contain.v.04"},{"index":5,"sense_key":"contain%2:42:14::","synset_id":"contain.v.05"},{"index":6,"sense_key":"contain%2:33:00::","synset_id":"check.v.18"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"contain%2:42:00::","call_ids":["semeval2015.d003.s003.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"contain%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":542,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001871,"input_uncached_usd":0.000027099999999999998,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.068859125021845}
{"item_id":"semeval2015.d003.s003.t001","gold_sense_keys":["active%3:00:07::"],"candidates":[{"index":1,"sense_key":"active%3:00:02::","synset_id":"active.a.01"},{"index":2,"sense_key":"active%5:00:00:operational:00","synset_id":"active.s.01"},{"index":3,"sense_key":"active%3:00:03::","synset_id":"active.a.03"},{"index":4,"sense_key":"active%5:00:00:involved:00","synset_id":"active.s.02"},{"index":5,"sense_key":"active%3:00:01::","synset_id":"active.a.05"},{"index":6,"sense_key":"active%3:00:07::","synset_id":"active.a.06"},{"index":7,"sense_key":"active%3:00:06::","synset_id":"active.a.07"},{"index":8,"sense_key":"active%5:00:00:existent:00","synset_id":"active.s.03"},{"index":9,"sense_key":"active%3:00:10::","synset_id":"active.a.09"},{"index":10,"sense_key":"active%3:00:09::","synset_id":"active.a.10"},{"index":11,"sense_key":"active%3:00:00::","synset_id":"active.a.11"},{"index":12,"sense_key":"active%3:00:05::","synset_id":"active.a.12"},{"index":13,"sense_key":"active%3:00:04::","synset_id":"active.a.13"},{"index":14,"sense_key":"active%3:00:08::","synset_id":"active.a.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":6,"chosen_sense_key":"active%3:00:07::","call_ids":["semeval2015.d003.s003.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":6,"predicted_sense_key":"active%3:00:07::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001481,"input_uncached_usd":0.0000393,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3436966669978574}
{"item_id":"semeval2015.d003.s003.t002","gold_sense_keys":["substance%1:03:01::"],"candidates":[{"index":1,"sense_key":"substance%1:03:00::","synset_id":"substance.n.01"},{"index":2,"sense_key":"substance%1:09:01::","synset_id":"kernel.n.03"},{"index":3,"sense_key":"substance%1:09:00::","synset_id":"meaning.n.02"},{"index":4,"sense_key":"substance%1:27:00::","synset_id":"substance.n.04"},{"index":5,"sense_key":"substance%1:21:00::","synset_id":"means.n.03"},{"index":6,"sense_key":"substance%1:10:00::","synset_id":"message.n.02"},{"index":7,"sense_key":"substance%1:03:01::","synset_id":"substance.n.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"substance%1:03:01::","call_ids":["semeval2015.d003.s003.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"substance%1:03:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":529,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054485,"input_uncached_usd":0.00002645,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.240586875006557}
{"item_id":"semeval2015.d003.s003.t003","gold_sense_keys":["available%3:00:00::"],"candidates":[{"index":1,"sense_key":"available%3:00:00::","synset_id":"available.a.01"},{"index":2,"sense_key":"available%5:00:00:free:00","synset_id":"available.s.01"},{"index":3,"sense_key":"available%5:00:00:disposable:02","synset_id":"available.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"available%3:00:00::","call_ids":["semeval2015.d003.s003.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"available%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":392,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015399999999999998,"input_uncached_usd":0.0000196,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.3344889580039307}
{"item_id":"semeval2015.d003.s003.t004","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s003.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":370,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025529999999999997,"input_uncached_usd":0.0000185,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.522397499997169}
{"item_id":"semeval2015.d003.s003.t005","gold_sense_keys":["solution%1:27:00::"],"candidates":[{"index":1,"sense_key":"solution%1:27:00::","synset_id":"solution.n.01"},{"index":2,"sense_key":"solution%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"solution%1:09:00::","synset_id":"solution.n.03"},{"index":4,"sense_key":"solution%1:14:00::","synset_id":"solution.n.04"},{"index":5,"sense_key":"solution%1:04:00::","synset_id":"solution.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"solution%1:27:00::","call_ids":["semeval2015.d003.s003.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"solution%1:27:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":447,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025915,"input_uncached_usd":0.000022349999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.317740541999228}
{"item_id":"semeval2015.d003.s003.t006","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"injection%1:27:00::","call_ids":["semeval2015.d003.s003.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"injection%1:27:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":344,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033079999999999996,"input_uncached_usd":0.000017199999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.455073207966052}
{"item_id":"semeval2015.d003.s004.t000","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s004.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":400,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.0004104,"input_uncached_usd":0.000019999999999999998,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.590514959010761}
{"item_id":"semeval2015.d003.s004.t001","gold_sense_keys":["have%2:40:00::","have%2:42:00::"],"candidates":[{"index":1,"sense_key":"have%2:40:00::","synset_id":"have.v.01"},{"index":2,"sense_key":"have%2:42:00::","synset_id":"have.v.02"},{"index":3,"sense_key":"have%2:39:06::","synset_id":"experience.v.03"},{"index":4,"sense_key":"have%2:40:04::","synset_id":"own.v.01"},{"index":5,"sense_key":"have%2:30:00::","synset_id":"get.v.03"},{"index":6,"sense_key":"have%2:34:00::","synset_id":"consume.v.02"},{"index":7,"sense_key":"have%2:41:00::","synset_id":"have.v.07"},{"index":8,"sense_key":"have%2:36:00::","synset_id":"hold.v.03"},{"index":9,"sense_key":"have%2:40:01::","synset_id":"have.v.09"},{"index":10,"sense_key":"have%2:42:12::","synset_id":"have.v.10"},{"index":11,"sense_key":"have%2:30:01::","synset_id":"have.v.11"},{"index":12,"sense_key":"have%2:29:02::","synset_id":"have.v.12"},{"index":13,"sense_key":"have%2:32:00::","synset_id":"induce.v.02"},{"index":14,"sense_key":"have%2:40:05::","synset_id":"accept.v.02"},{"index":15,"sense_key":"have%2:40:03::","synset_id":"receive.v.01"},{"index":16,"sense_key":"have%2:29:05::","synset_id":"suffer.v.02"},{"index":17,"sense_key":"have%2:40:02::","synset_id":"have.v.17"},{"index":18,"sense_key":"have%2:29:00::","synset_id":"give_birth.v.01"},{"index":19,"sense_key":"have%2:35:00::","synset_id":"take.v.35"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"have%2:42:00::","call_ids":["semeval2015.d003.s004.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"have%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1105,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00049685,"input_uncached_usd":0.000055249999999999994,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.558730999997351}
{"item_id":"semeval2015.d003.s004.t002","gold_sense_keys":["score%1:25:00::"],"candidates":[{"index":1,"sense_key":"score%1:09:00::","synset_id":"mark.n.01"},{"index":2,"sense_key":"score%1:10:00::","synset_id":"score.n.02"},{"index":3,"sense_key":"score%1:23:00::","synset_id":"score.n.03"},{"index":4,"sense_key":"score%1:14:00::","synset_id":"score.n.04"},{"index":5,"sense_key":"score%1:16:00::","synset_id":"score.n.05"},{"index":6,"sense_key":"score%1:09:01::","synset_id":"score.n.06"},{"index":7,"sense_key":"score%1:26:00::","synset_id":"score.n.07"},{"index":8,"sense_key":"score%1:25:00::","synset_id":"score.n.08"},{"index":9,"sense_key":"score%1:12:00::","synset_id":"grudge.n.01"},{"index":10,"sense_key":"score%1:04:00::","synset_id":"score.n.10"},{"index":11,"sense_key":"score%1:04:01::","synset_id":"sexual_conquest.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":8,"chosen_sense_key":"score%1:25:00::","call_ids":["semeval2015.d003.s004.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":8,"predicted_sense_key":"score%1:25:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":725,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042665,"input_uncached_usd":0.00003625,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.6337630419875495}
{"item_id":"semeval2015.d003.s004.t003","gold_sense_keys":["line%1:25:01::"],"candidates":[{"index":1,"sense_key":"line%1:14:03::","synset_id":"line.n.01"},{"index":2,"sense_key":"line%1:10:01::","synset_id":"line.n.02"},{"index":3,"sense_key":"line%1:14:01::","synset_id":"line.n.03"},{"index":4,"sense_key":"line%1:25:00::","synset_id":"line.n.04"},{"index":5,"sense_key":"line%1:10:02::","synset_id":"line.n.05"},{"index":6,"sense_key":"line%1:19:00::","synset_id":"line.n.06"},{"index":7,"sense_key":"line%1:15:02::","synset_id":"line.n.07"},{"index":8,"sense_key":"line%1:09:01::","synset_id":"argumentation.n.02"},{"index":9,"sense_key":"line%1:06:01::","synset_id":"cable.n.02"},{"index":10,"sense_key":"line%1:14:02::","synset_id":"course.n.02"},{"index":11,"sense_key":"line%1:15:00::","synset_id":"line.n.11"},{"index":12,"sense_key":"line%1:25:01::","synset_id":"wrinkle.n.01"},{"index":13,"sense_key":"line%1:06:06::","synset_id":"pipeline.n.02"},{"index":14,"sense_key":"line%1:06:03::","synset_id":"line.n.14"},{"index":15,"sense_key":"line%1:06:07::","synset_id":"telephone_line.n.02"},{"index":16,"sense_key":"line%1:04:01::","synset_id":"line.n.16"},{"index":17,"sense_key":"line%1:14:00::","synset_id":"lineage.n.01"},{"index":18,"sense_key":"line%1:06:00::","synset_id":"line.n.18"},{"index":19,"sense_key":"line%1:04:00::","synset_id":"occupation.n.01"},{"index":20,"sense_key":"line%1:15:01::","synset_id":"line.n.20"},{"index":21,"sense_key":"line%1:10:05::","synset_id":"channel.n.05"},{"index":22,"sense_key":"line%1:06:08::","synset_id":"line.n.22"},{"index":23,"sense_key":"line%1:06:05::","synset_id":"line.n.23"},{"index":24,"sense_key":"line%1:23:00::","synset_id":"agate_line.n.01"},{"index":25,"sense_key":"line%1:21:00::","synset_id":"credit_line.n.01"},{"index":26,"sense_key":"line%1:10:03::","synset_id":"tune.n.01"},{"index":27,"sense_key":"line%1:10:06::","synset_id":"line.n.27"},{"index":28,"sense_key":"line%1:10:04::","synset_id":"note.n.02"},{"index":29,"sense_key":"line%1:09:00::","synset_id":"line.n.29"},{"index":30,"sense_key":"line%1:06:09::","synset_id":"production_line.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"line%1:10:01::","call_ids":["semeval2015.d003.s004.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"line%1:10:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1516,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0006198,"input_uncached_usd":0.0000758,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.094087958044838}
{"item_id":"semeval2015.d003.s004.t004","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"side%1:15:00::","call_ids":["semeval2015.d003.s004.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"side%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":778,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00042929999999999997,"input_uncached_usd":0.0000389,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.969809791014995}
{"item_id":"semeval2015.d003.s004.t005","gold_sense_keys":["so%4:02:06::"],"candidates":[{"index":1,"sense_key":"so%4:02:02::","synset_id":"so.r.01"},{"index":2,"sense_key":"so%4:02:06::","synset_id":"so.r.02"},{"index":3,"sense_key":"so%4:02:05::","synset_id":"so.r.03"},{"index":4,"sense_key":"so%4:02:07::","synset_id":"so.r.04"},{"index":5,"sense_key":"so%4:02:08::","synset_id":"so.r.05"},{"index":6,"sense_key":"so%4:02:04::","synset_id":"thus.r.02"},{"index":7,"sense_key":"so%4:02:03::","synset_id":"so.r.07"},{"index":8,"sense_key":"so%4:02:09::","synset_id":"then.r.01"},{"index":9,"sense_key":"so%4:02:10::","synset_id":"therefore.r.01"},{"index":10,"sense_key":"so%4:02:00::","synset_id":"indeed.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"so%4:02:06::","call_ids":["semeval2015.d003.s004.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"so%4:02:06::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":752,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00035119999999999997,"input_uncached_usd":0.0000376,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.029377957980614}
{"item_id":"semeval2015.d003.s005.t000","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s005.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":407,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00043634999999999995,"input_uncached_usd":0.00002035,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.935791750031058}
{"item_id":"semeval2015.d003.s005.t001","gold_sense_keys":["marked%3:00:00::"],"candidates":[{"index":1,"sense_key":"marked%5:00:00:noticeable:00","synset_id":"marked.s.01"},{"index":2,"sense_key":"marked%5:00:00:conspicuous:00","synset_id":"marked.s.02"},{"index":3,"sense_key":"marked%3:00:00::","synset_id":"marked.a.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"marked%3:00:00::","call_ids":["semeval2015.d003.s005.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"marked%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":410,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001293,"input_uncached_usd":0.0000205,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.3557068750378676}
{"item_id":"semeval2015.d003.s005.t003","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"side%1:15:00::","call_ids":["semeval2015.d003.s005.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"side%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":786,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.0002761,"input_uncached_usd":0.0000393,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.412335916014854}
{"item_id":"semeval2015.d003.s005.t004","gold_sense_keys":["letter%1:10:01::"],"candidates":[{"index":1,"sense_key":"letter%1:10:00::","synset_id":"letter.n.01"},{"index":2,"sense_key":"letter%1:10:01::","synset_id":"letter.n.02"},{"index":3,"sense_key":"letter%1:18:00::","synset_id":"letter.n.03"},{"index":4,"sense_key":"letter%1:10:03::","synset_id":"letter.n.04"},{"index":5,"sense_key":"letter%1:10:02::","synset_id":"letter.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"letter%1:10:01::","call_ids":["semeval2015.d003.s005.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"letter%1:10:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":482,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005681,"input_uncached_usd":0.0000241,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.069924666022416}
{"item_id":"semeval2015.d003.s005.t005","gold_sense_keys":["quantity%1:03:00::"],"candidates":[{"index":1,"sense_key":"quantity%1:03:00::","synset_id":"measure.n.02"},{"index":2,"sense_key":"quantity%1:07:00::","synset_id":"quantity.n.02"},{"index":3,"sense_key":"quantity%1:09:01::","synset_id":"quantity.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"quantity%1:03:00::","call_ids":["semeval2015.d003.s005.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"quantity%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015355,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.858299166953657}
{"item_id":"semeval2015.d003.s005.t006","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s005.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":451,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020815,"input_uncached_usd":0.00002255,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.694163625012152}
{"item_id":"semeval2015.d003.s005.t007","gold_sense_keys":["side%1:06:00::","side%1:15:00::"],"candidates":[{"index":1,"sense_key":"side%1:15:02::","synset_id":"side.n.01"},{"index":2,"sense_key":"side%1:14:00::","synset_id":"side.n.02"},{"index":3,"sense_key":"side%1:08:00::","synset_id":"side.n.03"},{"index":4,"sense_key":"side%1:15:00::","synset_id":"side.n.04"},{"index":5,"sense_key":"side%1:06:00::","synset_id":"side.n.05"},{"index":6,"sense_key":"side%1:09:00::","synset_id":"side.n.06"},{"index":7,"sense_key":"side%1:15:01::","synset_id":"side.n.07"},{"index":8,"sense_key":"side%1:14:01::","synset_id":"side.n.08"},{"index":9,"sense_key":"side%1:13:00::","synset_id":"side.n.09"},{"index":10,"sense_key":"side%1:10:00::","synset_id":"side.n.10"},{"index":11,"sense_key":"side%1:17:01::","synset_id":"slope.n.01"},{"index":12,"sense_key":"side%1:11:00::","synset_id":"english.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"side%1:15:00::","call_ids":["semeval2015.d003.s005.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"side%1:15:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":785,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00032725,"input_uncached_usd":0.00003925,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.350001709011849}
{"item_id":"semeval2015.d003.s006.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s006.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":563,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005209500000000001,"input_uncached_usd":0.00002815,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.806287457991857}
{"item_id":"semeval2015.d003.s007.t001","gold_sense_keys":["mean%2:32:03::","mean%2:42:00::"],"candidates":[{"index":1,"sense_key":"mean%2:32:01::","synset_id":"mean.v.01"},{"index":2,"sense_key":"mean%2:42:00::","synset_id":"entail.v.01"},{"index":3,"sense_key":"mean%2:32:03::","synset_id":"mean.v.03"},{"index":4,"sense_key":"mean%2:31:00::","synset_id":"intend.v.01"},{"index":5,"sense_key":"mean%2:42:03::","synset_id":"mean.v.05"},{"index":6,"sense_key":"mean%2:31:01::","synset_id":"think_of.v.04"},{"index":7,"sense_key":"mean%2:31:05::","synset_id":"mean.v.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"mean%2:42:00::","call_ids":["semeval2015.d003.s007.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"mean%2:42:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":595,"cached_input_tokens":0,"output_tokens":1616,"reasoning_output_tokens":1600},"cost":{"currency":"USD","total_usd":0.00067615,"input_uncached_usd":0.000029749999999999998,"input_cached_usd":0.0,"output_usd":0.0006464,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.203761207987554}
{"item_id":"semeval2015.d003.s007.t002","gold_sense_keys":["stop%2:41:00::"],"candidates":[{"index":1,"sense_key":"stop%2:38:00::","synset_id":"stop.v.01"},{"index":2,"sense_key":"stop%2:42:00::","synset_id":"discontinue.v.01"},{"index":3,"sense_key":"stop%2:41:00::","synset_id":"stop.v.03"},{"index":4,"sense_key":"stop%2:38:02::","synset_id":"stop.v.04"},{"index":5,"sense_key":"stop%2:38:01::","synset_id":"stop.v.05"},{"index":6,"sense_key":"stop%2:30:05::","synset_id":"break.v.10"},{"index":7,"sense_key":"stop%2:33:00::","synset_id":"check.v.18"},{"index":8,"sense_key":"stop%2:35:00::","synset_id":"intercept.v.01"},{"index":9,"sense_key":"stop%2:42:13::","synset_id":"end.v.01"},{"index":10,"sense_key":"stop%2:35:12::","synset_id":"barricade.v.01"},{"index":11,"sense_key":"stop%2:30:12::","synset_id":"hold_on.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"stop%2:42:00::","call_ids":["semeval2015.d003.s007.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"stop%2:42:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":802,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006353000000000001,"input_uncached_usd":0.0000401,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.978922542009968}
{"item_id":"semeval2015.d003.s008.t000","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s008.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.00077865,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.573814583010972}
{"item_id":"semeval2015.d003.s008.t001","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s008.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":654,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014149999999999997,"input_uncached_usd":0.000032699999999999995,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6080417500343174}
{"item_id":"semeval2015.d003.s008.t002","gold_sense_keys":["prevent%2:41:01::"],"candidates":[{"index":1,"sense_key":"prevent%2:41:01::","synset_id":"prevent.v.01"},{"index":2,"sense_key":"prevent%2:41:00::","synset_id":"prevent.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"prevent%2:41:00::","call_ids":["semeval2015.d003.s008.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"prevent%2:41:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":434,"cached_input_tokens":0,"output_tokens":1488,"reasoning_output_tokens":1472},"cost":{"currency":"USD","total_usd":0.0006169000000000001,"input_uncached_usd":0.0000217,"input_cached_usd":0.0,"output_usd":0.0005951999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.800144458014984}
{"item_id":"semeval2015.d003.s008.t005","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d003.s008.t005__v1__a1","semeval2015.d003.s008.t005__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":899,"cached_input_tokens":0,"output_tokens":1440,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.0006209499999999999,"input_uncached_usd":0.000044949999999999995,"input_cached_usd":0.0,"output_usd":0.000576,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.134338833973743}
{"item_id":"semeval2015.d003.s008.t007","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"medicine%1:04:00::","call_ids":["semeval2015.d003.s008.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"medicine%1:04:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":1168,"reasoning_output_tokens":1152},"cost":{"currency":"USD","total_usd":0.00049155,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.00046719999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.04054733295925}
{"item_id":"semeval2015.d003.s008.t008","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s008.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":597,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036904999999999994,"input_uncached_usd":0.000029849999999999997,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.60675304196775}
{"item_id":"semeval2015.d003.s008.t009","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"treatment%1:07:00::","call_ids":["semeval2015.d003.s008.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"treatment%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":492,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.0003894,"input_uncached_usd":0.000024599999999999998,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.136455291009042}
{"item_id":"semeval2015.d003.s008.t010","gold_sense_keys":["cancer%1:26:00::"],"candidates":[{"index":1,"sense_key":"cancer%1:26:00::","synset_id":"cancer.n.01"},{"index":2,"sense_key":"cancer%1:18:00::","synset_id":"cancer.n.02"},{"index":3,"sense_key":"cancer%1:17:00::","synset_id":"cancer.n.03"},{"index":4,"sense_key":"cancer%1:15:00::","synset_id":"cancer.n.04"},{"index":5,"sense_key":"cancer%1:05:00::","synset_id":"cancer.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cancer%1:26:00::","call_ids":["semeval2015.d003.s008.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cancer%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":515,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023694999999999999,"input_uncached_usd":0.00002575,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.9000188339850865}
{"item_id":"semeval2015.d003.s008.t012","gold_sense_keys":["cause%2:36:00::"],"candidates":[{"index":1,"sense_key":"cause%2:36:00::","synset_id":"cause.v.01"},{"index":2,"sense_key":"cause%2:32:00::","synset_id":"induce.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"cause%2:36:00::","call_ids":["semeval2015.d003.s008.t012__v1__a1","semeval2015.d003.s008.t012__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"cause%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":897,"cached_input_tokens":0,"output_tokens":1696,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00072325,"input_uncached_usd":0.00004484999999999999,"input_cached_usd":0.0,"output_usd":0.0006783999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.341517124965321}
{"item_id":"semeval2015.d003.s009.t002","gold_sense_keys":["measure%1:04:01::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"measure%1:04:01::","call_ids":["semeval2015.d003.s009.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"measure%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":710,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00022109999999999998,"input_uncached_usd":0.000035499999999999996,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.742525374982506}
{"item_id":"semeval2015.d003.s009.t003","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s009.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":484,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000261,"input_uncached_usd":0.0000242,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.079981832997873}
{"item_id":"semeval2015.d003.s009.t005","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s009.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031235000000000003,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.71617208403768}
{"item_id":"semeval2015.d003.s009.t006","gold_sense_keys":["special%5:00:00:specialized:00","special%5:00:01:specific:00"],"candidates":[{"index":1,"sense_key":"special%5:00:01:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"special%5:00:00:extraordinary:00","synset_id":"special.s.02"},{"index":3,"sense_key":"special%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"special%5:00:00:specialized:00","synset_id":"special.s.04"},{"index":5,"sense_key":"special%5:00:00:specific:00","synset_id":"limited.s.05"},{"index":6,"sense_key":"special%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":7,"sense_key":"special%5:00:00:unscheduled:00","synset_id":"extra.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"special%5:00:00:specialized:00","call_ids":["semeval2015.d003.s009.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"special%5:00:00:specialized:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":633,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044764999999999996,"input_uncached_usd":0.00003165,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.941548833041452}
{"item_id":"semeval2015.d003.s009.t007","gold_sense_keys":["diet%1:13:00::"],"candidates":[{"index":1,"sense_key":"diet%1:13:00::","synset_id":"diet.n.01"},{"index":2,"sense_key":"diet%1:14:00::","synset_id":"diet.n.02"},{"index":3,"sense_key":"diet%1:13:01::","synset_id":"diet.n.03"},{"index":4,"sense_key":"diet%1:04:00::","synset_id":"diet.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"diet%1:13:00::","call_ids":["semeval2015.d003.s009.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"diet%1:13:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":424,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.0003604,"input_uncached_usd":0.0000212,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.778710792015772}
{"item_id":"semeval2015.d003.s009.t009","gold_sense_keys":["use%2:34:01::"],"candidates":[{"index":1,"sense_key":"use%2:34:01::","synset_id":"use.v.01"},{"index":2,"sense_key":"use%2:34:02::","synset_id":"use.v.02"},{"index":3,"sense_key":"use%2:34:00::","synset_id":"use.v.03"},{"index":4,"sense_key":"use%2:41:14::","synset_id":"use.v.04"},{"index":5,"sense_key":"use%2:41:04::","synset_id":"practice.v.04"},{"index":6,"sense_key":"use%2:41:03::","synset_id":"use.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"use%2:34:01::","call_ids":["semeval2015.d003.s009.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"use%2:34:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":592,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003176,"input_uncached_usd":0.000029599999999999998,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.398284333990887}
{"item_id":"semeval2015.d003.s009.t010","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s009.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":487,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028675,"input_uncached_usd":0.00002435,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.49815487494925}
{"item_id":"semeval2015.d003.s009.t012","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s009.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020634999999999997,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.566702832991723}
{"item_id":"semeval2015.d003.s009.t013","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s009.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":415,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015514999999999998,"input_uncached_usd":0.00002075,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9574502920149826}
{"item_id":"semeval2015.d003.s009.t015","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s009.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":440,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038679999999999997,"input_uncached_usd":0.000022,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.354201957990881}
{"item_id":"semeval2015.d003.s010.t000","gold_sense_keys":["purpose%1:07:00::"],"candidates":[{"index":1,"sense_key":"purpose%1:09:00::","synset_id":"purpose.n.01"},{"index":2,"sense_key":"purpose%1:07:00::","synset_id":"function.n.02"},{"index":3,"sense_key":"purpose%1:07:01::","synset_id":"determination.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"purpose%1:07:00::","call_ids":["semeval2015.d003.s010.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"purpose%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":455,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00025955,"input_uncached_usd":0.000022749999999999997,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.447384209022857}
{"item_id":"semeval2015.d003.s010.t001","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d003.s010.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2044,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00023659999999999998,"input_uncached_usd":0.00010219999999999999,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.827847042004578}
{"item_id":"semeval2015.d003.s010.t002","gold_sense_keys":["day%1:28:00::"],"candidates":[{"index":1,"sense_key":"day%1:28:00::","synset_id":"day.n.01"},{"index":2,"sense_key":"day%1:28:03::","synset_id":"day.n.02"},{"index":3,"sense_key":"day%1:28:01::","synset_id":"day.n.03"},{"index":4,"sense_key":"day%1:28:02::","synset_id":"day.n.04"},{"index":5,"sense_key":"day%1:28:04::","synset_id":"day.n.05"},{"index":6,"sense_key":"day%1:28:05::","synset_id":"day.n.06"},{"index":7,"sense_key":"day%1:28:07::","synset_id":"day.n.07"},{"index":8,"sense_key":"day%1:28:06::","synset_id":"sidereal_day.n.01"},{"index":9,"sense_key":"day%1:26:00::","synset_id":"day.n.09"},{"index":10,"sense_key":"day%1:18:00::","synset_id":"day.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"day%1:28:00::","call_ids":["semeval2015.d003.s010.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"day%1:28:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":759,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00019795,"input_uncached_usd":0.00003795,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9831953329849057}
{"item_id":"semeval2015.d003.s012.t001","gold_sense_keys":["receptor%1:08:01::"],"candidates":[{"index":1,"sense_key":"receptor%1:08:01::","synset_id":"receptor.n.01"},{"index":2,"sense_key":"receptor%1:08:00::","synset_id":"sense_organ.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"receptor%1:08:01::","call_ids":["semeval2015.d003.s012.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"receptor%1:08:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":383,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033275,"input_uncached_usd":0.000019149999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.041759874962736}
{"item_id":"semeval2015.d003.s013.t000","gold_sense_keys":["study%2:31:02::"],"candidates":[{"index":1,"sense_key":"study%2:31:02::","synset_id":"analyze.v.01"},{"index":2,"sense_key":"study%2:31:03::","synset_id":"study.v.02"},{"index":3,"sense_key":"study%2:39:00::","synset_id":"study.v.03"},{"index":4,"sense_key":"study%2:31:00::","synset_id":"learn.v.04"},{"index":5,"sense_key":"study%2:31:01::","synset_id":"study.v.05"},{"index":6,"sense_key":"study%2:31:04::","synset_id":"study.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"study%2:31:02::","call_ids":["semeval2015.d003.s013.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"study%2:31:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":577,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00041925,"input_uncached_usd":0.00002885,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.980463458981831}
{"item_id":"semeval2015.d003.s014.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d003.s014.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":699,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00034855,"input_uncached_usd":0.000034949999999999996,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.197705959028099}
{"item_id":"semeval2015.d003.s014.t002","gold_sense_keys":["carry_out%2:36:00::"],"candidates":[{"index":1,"sense_key":"carry_out%2:36:00::","synset_id":"carry_through.v.01"},{"index":2,"sense_key":"carry_out%2:30:00::","synset_id":"follow_through.v.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"carry_out%2:36:00::","call_ids":["semeval2015.d003.s014.t002__v1__a1","semeval2015.d003.s014.t002__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"carry_out%2:36:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":853,"cached_input_tokens":0,"output_tokens":1376,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005930499999999999,"input_uncached_usd":0.00004264999999999999,"input_cached_usd":0.0,"output_usd":0.0005503999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.280380499956664}
{"item_id":"semeval2015.d003.s014.t003","gold_sense_keys":["laboratory%1:06:00::"],"candidates":[{"index":1,"sense_key":"laboratory%1:06:00::","synset_id":"lab.n.01"},{"index":2,"sense_key":"laboratory%1:15:00::","synset_id":"testing_ground.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"laboratory%1:06:00::","call_ids":["semeval2015.d003.s014.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"laboratory%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":394,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.00033329999999999997,"input_uncached_usd":0.000019699999999999998,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.775798083981499}
{"item_id":"semeval2015.d003.s014.t004","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s014.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":623,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037035,"input_uncached_usd":0.00003115,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.043030083004851}
{"item_id":"semeval2015.d003.s014.t007","gold_sense_keys":["several%5:00:00:some:00"],"candidates":[{"index":1,"sense_key":"several%5:00:00:some:00","synset_id":"several.s.01"},{"index":2,"sense_key":"several%5:00:00:individual:00","synset_id":"respective.s.01"},{"index":3,"sense_key":"several%5:00:00:different:00","synset_id":"several.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"several%5:00:00:some:00","call_ids":["semeval2015.d003.s014.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"several%5:00:00:some:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023199999999999997,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5407151670078747}
{"item_id":"semeval2015.d003.s015.t000","gold_sense_keys":["benefit%1:07:00::"],"candidates":[{"index":1,"sense_key":"benefit%1:21:00::","synset_id":"benefit.n.01"},{"index":2,"sense_key":"benefit%1:07:00::","synset_id":"benefit.n.02"},{"index":3,"sense_key":"benefit%1:10:00::","synset_id":"benefit.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"benefit%1:07:00::","call_ids":["semeval2015.d003.s015.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"benefit%1:07:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":380,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020459999999999999,"input_uncached_usd":0.000018999999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0644820840097964}
{"item_id":"semeval2015.d003.s015.t001","gold_sense_keys":["show%2:32:00::","show%2:39:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"show%2:31:00::","call_ids":["semeval2015.d003.s015.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"show%2:31:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":862,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.0004591,"input_uncached_usd":0.0000431,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.706131834012922}
{"item_id":"semeval2015.d003.s015.t002","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d003.s015.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":706,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.0003233,"input_uncached_usd":0.0000353,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.676477542030625}
{"item_id":"semeval2015.d003.s016.t000","gold_sense_keys":["result%1:11:00::"],"candidates":[{"index":1,"sense_key":"result%1:19:00::","synset_id":"consequence.n.01"},{"index":2,"sense_key":"result%1:10:00::","synset_id":"solution.n.02"},{"index":3,"sense_key":"result%1:11:00::","synset_id":"result.n.03"},{"index":4,"sense_key":"result%1:10:01::","synset_id":"resultant_role.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"result%1:19:00::","call_ids":["semeval2015.d003.s016.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"result%1:19:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":523,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018615,"input_uncached_usd":0.00002615,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.902792875014711}
{"item_id":"semeval2015.d003.s016.t001","gold_sense_keys":["study%1:04:00::"],"candidates":[{"index":1,"sense_key":"study%1:04:00::","synset_id":"survey.n.01"},{"index":2,"sense_key":"study%1:09:03::","synset_id":"study.n.02"},{"index":3,"sense_key":"study%1:10:00::","synset_id":"report.n.01"},{"index":4,"sense_key":"study%1:09:04::","synset_id":"study.n.04"},{"index":5,"sense_key":"study%1:06:00::","synset_id":"study.n.05"},{"index":6,"sense_key":"study%1:09:02::","synset_id":"discipline.n.01"},{"index":7,"sense_key":"study%1:06:01::","synset_id":"sketch.n.01"},{"index":8,"sense_key":"study%1:09:01::","synset_id":"cogitation.n.02"},{"index":9,"sense_key":"study%1:18:00::","synset_id":"study.n.09"},{"index":10,"sense_key":"study%1:10:01::","synset_id":"study.n.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"study%1:10:00::","call_ids":["semeval2015.d003.s016.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"study%1:10:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":728,"cached_input_tokens":0,"output_tokens":2384,"reasoning_output_tokens":2368},"cost":{"currency":"USD","total_usd":0.00099,"input_uncached_usd":0.0000364,"input_cached_usd":0.0,"output_usd":0.0009536,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.015305250009988}
{"item_id":"semeval2015.d003.s016.t002","gold_sense_keys":["show%2:31:00::","show%2:32:00::"],"candidates":[{"index":1,"sense_key":"show%2:39:02::","synset_id":"show.v.01"},{"index":2,"sense_key":"show%2:31:00::","synset_id":"prove.v.02"},{"index":3,"sense_key":"show%2:32:00::","synset_id":"testify.v.02"},{"index":4,"sense_key":"show%2:39:00::","synset_id":"show.v.04"},{"index":5,"sense_key":"show%2:36:00::","synset_id":"picture.v.02"},{"index":6,"sense_key":"show%2:32:02::","synset_id":"express.v.01"},{"index":7,"sense_key":"show%2:32:01::","synset_id":"indicate.v.02"},{"index":8,"sense_key":"show%2:39:01::","synset_id":"show.v.08"},{"index":9,"sense_key":"show%2:32:04::","synset_id":"read.v.08"},{"index":10,"sense_key":"show%2:32:03::","synset_id":"show.v.10"},{"index":11,"sense_key":"show%2:38:00::","synset_id":"usher.v.01"},{"index":12,"sense_key":"show%2:33:00::","synset_id":"show.v.12"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"show%2:31:00::","call_ids":["semeval2015.d003.s016.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"show%2:31:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":884,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.0002554,"input_uncached_usd":0.0000442,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.6519351250026375}
{"item_id":"semeval2015.d003.s016.t003","gold_sense_keys":["more%4:02:00::"],"candidates":[{"index":1,"sense_key":"more%4:02:00::","synset_id":"more.r.01"},{"index":2,"sense_key":"more%4:02:01::","synset_id":"more.r.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"more%4:02:00::","call_ids":["semeval2015.d003.s016.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"more%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":398,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005127,"input_uncached_usd":0.0000199,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":9.43771004199516}
{"item_id":"semeval2015.d003.s016.t004","gold_sense_keys":["effective%3:00:00::","effective%5:00:00:competent:00","effective%5:00:00:efficacious:00"],"candidates":[{"index":1,"sense_key":"effective%3:00:00::","synset_id":"effective.a.01"},{"index":2,"sense_key":"effective%5:00:00:competent:00","synset_id":"effective.s.01"},{"index":3,"sense_key":"effective%5:00:00:efficacious:00","synset_id":"effective.s.02"},{"index":4,"sense_key":"effective%5:00:00:operative:00","synset_id":"effective.s.03"},{"index":5,"sense_key":"effective%5:00:00:actual:00","synset_id":"effective.s.04"},{"index":6,"sense_key":"effective%5:00:00:operational:00","synset_id":"effective.s.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"effective%5:00:00:efficacious:00","call_ids":["semeval2015.d003.s016.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"effective%5:00:00:efficacious:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":634,"cached_input_tokens":0,"output_tokens":1232,"reasoning_output_tokens":1216},"cost":{"currency":"USD","total_usd":0.0005245,"input_uncached_usd":0.0000317,"input_cached_usd":0.0,"output_usd":0.0004927999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":11.256837499968242}
{"item_id":"semeval2015.d003.s016.t005","gold_sense_keys":["placebo%1:06:00::"],"candidates":[{"index":1,"sense_key":"placebo%1:06:00::","synset_id":"placebo.n.01"},{"index":2,"sense_key":"placebo%1:04:00::","synset_id":"placebo.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"placebo%1:06:00::","call_ids":["semeval2015.d003.s016.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"placebo%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":387,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015375,"input_uncached_usd":0.00001935,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.587362333026249}
{"item_id":"semeval2015.d003.s016.t006","gold_sense_keys":["less%3:00:00::"],"candidates":[{"index":1,"sense_key":"less%3:00:00::","synset_id":"less.a.01"},{"index":2,"sense_key":"less%5:00:00:inferior:02","synset_id":"less.s.01"},{"index":3,"sense_key":"less%5:00:00:fewer:00","synset_id":"less.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"less%3:00:00::","call_ids":["semeval2015.d003.s016.t006__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"less%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.000157,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.9082204579608515}
{"item_id":"semeval2015.d003.s016.t008","gold_sense_keys":["see%2:39:02::"],"candidates":[{"index":1,"sense_key":"see%2:39:00::","synset_id":"see.v.01"},{"index":2,"sense_key":"see%2:31:01::","synset_id":"understand.v.02"},{"index":3,"sense_key":"see%2:39:02::","synset_id":"witness.v.02"},{"index":4,"sense_key":"see%2:36:00::","synset_id":"visualize.v.01"},{"index":5,"sense_key":"see%2:31:00::","synset_id":"see.v.05"},{"index":6,"sense_key":"see%2:31:02::","synset_id":"learn.v.02"},{"index":7,"sense_key":"see%2:39:03::","synset_id":"watch.v.03"},{"index":8,"sense_key":"see%2:38:00::","synset_id":"meet.v.01"},{"index":9,"sense_key":"see%2:32:00::","synset_id":"determine.v.08"},{"index":10,"sense_key":"see%2:31:03::","synset_id":"see.v.10"},{"index":11,"sense_key":"see%2:41:11::","synset_id":"see.v.11"},{"index":12,"sense_key":"see%2:41:12::","synset_id":"see.v.12"},{"index":13,"sense_key":"see%2:41:10::","synset_id":"visit.v.01"},{"index":14,"sense_key":"see%2:41:14::","synset_id":"attend.v.02"},{"index":15,"sense_key":"see%2:35:09::","synset_id":"see.v.15"},{"index":16,"sense_key":"see%2:41:00::","synset_id":"go_steady.v.01"},{"index":17,"sense_key":"see%2:39:05::","synset_id":"see.v.17"},{"index":18,"sense_key":"see%2:31:08::","synset_id":"see.v.18"},{"index":19,"sense_key":"see%2:39:11::","synset_id":"see.v.19"},{"index":20,"sense_key":"see%2:39:13::","synset_id":"examine.v.02"},{"index":21,"sense_key":"see%2:39:12::","synset_id":"experience.v.01"},{"index":22,"sense_key":"see%2:38:03::","synset_id":"see.v.22"},{"index":23,"sense_key":"see%2:33:00::","synset_id":"see.v.23"},{"index":24,"sense_key":"see%2:31:13::","synset_id":"interpret.v.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":7,"chosen_sense_key":"see%2:39:03::","call_ids":["semeval2015.d003.s016.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":7,"predicted_sense_key":"see%2:39:03::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":1457,"cached_input_tokens":0,"output_tokens":976,"reasoning_output_tokens":960},"cost":{"currency":"USD","total_usd":0.00046325,"input_uncached_usd":0.00007285,"input_cached_usd":0.0,"output_usd":0.0003904,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.684579332999419}
{"item_id":"semeval2015.d003.s016.t009","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s016.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.0001414,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.268622791976668}
{"item_id":"semeval2015.d003.s016.t011","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s016.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":486,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.0002867,"input_uncached_usd":0.000024299999999999998,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.57191237504594}
{"item_id":"semeval2015.d003.s016.t012","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s016.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":652,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.000167,"input_uncached_usd":0.0000326,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.374821500037797}
{"item_id":"semeval2015.d003.s016.t014","gold_sense_keys":["placebo%1:06:00::"],"candidates":[{"index":1,"sense_key":"placebo%1:06:00::","synset_id":"placebo.n.01"},{"index":2,"sense_key":"placebo%1:04:00::","synset_id":"placebo.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"placebo%1:06:00::","call_ids":["semeval2015.d003.s016.t014__v1__a1","semeval2015.d003.s016.t014__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"placebo%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":821,"cached_input_tokens":0,"output_tokens":1056,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00046345,"input_uncached_usd":0.000041049999999999995,"input_cached_usd":0.0,"output_usd":0.00042239999999999997,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.490645750018302}
{"item_id":"semeval2015.d003.s016.t015","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"treatment%1:07:00::","call_ids":["semeval2015.d003.s016.t015__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"treatment%1:07:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":490,"cached_input_tokens":0,"output_tokens":1552,"reasoning_output_tokens":1536},"cost":{"currency":"USD","total_usd":0.0006453,"input_uncached_usd":0.0000245,"input_cached_usd":0.0,"output_usd":0.0006208,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":12.104131292027887}
{"item_id":"semeval2015.d003.s017.t000","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s017.t000__v1__a1","semeval2015.d003.s017.t000__v1__a2"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":1045,"cached_input_tokens":0,"output_tokens":2912,"reasoning_output_tokens":2880},"cost":{"currency":"USD","total_usd":0.0012170500000000001,"input_uncached_usd":0.000052249999999999996,"input_cached_usd":0.0,"output_usd":0.0011648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":17.725301500002388}
{"item_id":"semeval2015.d003.s017.t002","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s017.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":661,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00026984999999999997,"input_uncached_usd":0.00003305,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8157636250252835}
{"item_id":"semeval2015.d003.s017.t003","gold_sense_keys":["start%2:30:01::","start%2:41:04::"],"candidates":[{"index":1,"sense_key":"start%2:30:00::","synset_id":"get_down.v.07"},{"index":2,"sense_key":"start%2:30:01::","synset_id":"begin.v.03"},{"index":3,"sense_key":"start%2:38:02::","synset_id":"depart.v.03"},{"index":4,"sense_key":"start%2:42:00::","synset_id":"begin.v.02"},{"index":5,"sense_key":"start%2:36:00::","synset_id":"originate.v.02"},{"index":6,"sense_key":"start%2:36:01::","synset_id":"start.v.06"},{"index":7,"sense_key":"start%2:38:04::","synset_id":"startle.v.02"},{"index":8,"sense_key":"start%2:38:01::","synset_id":"start.v.08"},{"index":9,"sense_key":"start%2:38:00::","synset_id":"start.v.09"},{"index":10,"sense_key":"start%2:41:00::","synset_id":"start.v.10"},{"index":11,"sense_key":"start%2:33:00::","synset_id":"start.v.11"},{"index":12,"sense_key":"start%2:42:03::","synset_id":"begin.v.07"},{"index":13,"sense_key":"start%2:41:04::","synset_id":"begin.v.08"},{"index":14,"sense_key":"start%2:38:03::","synset_id":"start.v.14"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":9,"chosen_sense_key":"start%2:38:00::","call_ids":["semeval2015.d003.s017.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":9,"predicted_sense_key":"start%2:38:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":949,"cached_input_tokens":0,"output_tokens":2192,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0009242499999999999,"input_uncached_usd":0.000047449999999999995,"input_cached_usd":0.0,"output_usd":0.0008768,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.798822083044797}
{"item_id":"semeval2015.d003.s017.t004","gold_sense_keys":["injection%1:04:01::"],"candidates":[{"index":1,"sense_key":"injection%1:04:00::","synset_id":"injection.n.01"},{"index":2,"sense_key":"injection%1:27:00::","synset_id":"injection.n.02"},{"index":3,"sense_key":"injection%1:04:01::","synset_id":"injection.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"injection%1:04:01::","call_ids":["semeval2015.d003.s017.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"injection%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":426,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.0001813,"input_uncached_usd":0.0000213,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.4972330419695936}
{"item_id":"semeval2015.d003.s017.t008","gold_sense_keys":["able%3:00:00::"],"candidates":[{"index":1,"sense_key":"able%3:00:00::","synset_id":"able.a.01"},{"index":2,"sense_key":"able%5:00:00:competent:00","synset_id":"able.s.01"},{"index":3,"sense_key":"able%5:00:00:capable:00","synset_id":"able.s.02"},{"index":4,"sense_key":"able%5:00:00:fit:01","synset_id":"able.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"able%5:00:00:capable:00","call_ids":["semeval2015.d003.s017.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"able%5:00:00:capable:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":509,"cached_input_tokens":0,"output_tokens":528,"reasoning_output_tokens":512},"cost":{"currency":"USD","total_usd":0.00023664999999999998,"input_uncached_usd":0.00002545,"input_cached_usd":0.0,"output_usd":0.00021119999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.645442333945539}
{"item_id":"semeval2015.d003.s017.t009","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s017.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.0002082,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.8341102090198547}
{"item_id":"semeval2015.d003.s017.t010","gold_sense_keys":["intact%5:00:00:undamaged:00","intact%5:00:00:whole:00"],"candidates":[{"index":1,"sense_key":"intact%5:00:00:whole:00","synset_id":"integral.s.02"},{"index":2,"sense_key":"intact%5:00:00:uninjured:00","synset_id":"intact.s.02"},{"index":3,"sense_key":"intact%5:00:00:uncastrated:00","synset_id":"entire.s.04"},{"index":4,"sense_key":"intact%5:00:00:undamaged:00","synset_id":"intact.s.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"intact%5:00:00:undamaged:00","call_ids":["semeval2015.d003.s017.t010__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"intact%5:00:00:undamaged:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":510,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031350000000000003,"input_uncached_usd":0.0000255,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.078393124975264}
{"item_id":"semeval2015.d003.s017.t011","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s017.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":1104,"reasoning_output_tokens":1088},"cost":{"currency":"USD","total_usd":0.00046419999999999995,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0004416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.151866499974858}
{"item_id":"semeval2015.d003.s018.t000","gold_sense_keys":["follow-up%1:04:00::","follow-up%1:04:01::"],"candidates":[{"index":1,"sense_key":"follow-up%1:06:00::","synset_id":"follow-up.n.01"},{"index":2,"sense_key":"follow-up%1:04:01::","synset_id":"follow-up.n.02"},{"index":3,"sense_key":"follow-up%1:04:00::","synset_id":"follow-up.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"follow-up%1:04:01::","call_ids":["semeval2015.d003.s018.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"follow-up%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":444,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.000259,"input_uncached_usd":0.000022199999999999998,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5830803340068087}
{"item_id":"semeval2015.d003.s018.t001","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s018.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044104999999999996,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.127794499974698}
{"item_id":"semeval2015.d003.s018.t002","gold_sense_keys":["owner%1:18:02::"],"candidates":[{"index":1,"sense_key":"owner%1:18:00::","synset_id":"owner.n.01"},{"index":2,"sense_key":"owner%1:18:02::","synset_id":"owner.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"owner%1:18:02::","call_ids":["semeval2015.d003.s018.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"owner%1:18:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":421,"cached_input_tokens":0,"output_tokens":336,"reasoning_output_tokens":320},"cost":{"currency":"USD","total_usd":0.00015544999999999998,"input_uncached_usd":0.00002105,"input_cached_usd":0.0,"output_usd":0.0001344,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6261194580001757}
{"item_id":"semeval2015.d003.s018.t003","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"},{"index":2,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":3,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":4,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":5,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"administer%2:29:00::","call_ids":["semeval2015.d003.s018.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":533,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018664999999999998,"input_uncached_usd":0.000026649999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.815358208026737}
{"item_id":"semeval2015.d003.s018.t004","gold_sense_keys":["tablet%1:06:00::"],"candidates":[{"index":1,"sense_key":"tablet%1:06:02::","synset_id":"tablet.n.01"},{"index":2,"sense_key":"tablet%1:27:00::","synset_id":"pad.n.01"},{"index":3,"sense_key":"tablet%1:06:01::","synset_id":"tablet.n.03"},{"index":4,"sense_key":"tablet%1:06:00::","synset_id":"pill.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"tablet%1:06:00::","call_ids":["semeval2015.d003.s018.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"tablet%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":456,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00020839999999999997,"input_uncached_usd":0.0000228,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.325674332969356}
{"item_id":"semeval2015.d003.s018.t005","gold_sense_keys":["dog%1:05:00::"],"candidates":[{"index":1,"sense_key":"dog%1:05:00::","synset_id":"dog.n.01"},{"index":2,"sense_key":"dog%1:18:01::","synset_id":"frump.n.01"},{"index":3,"sense_key":"dog%1:18:00::","synset_id":"dog.n.03"},{"index":4,"sense_key":"dog%1:18:02::","synset_id":"cad.n.01"},{"index":5,"sense_key":"dog%1:13:01::","synset_id":"frank.n.02"},{"index":6,"sense_key":"dog%1:06:00::","synset_id":"pawl.n.01"},{"index":7,"sense_key":"dog%1:06:01::","synset_id":"andiron.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dog%1:05:00::","call_ids":["semeval2015.d003.s018.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dog%1:05:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":663,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00014194999999999998,"input_uncached_usd":0.00003315,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.369536040991079}
{"item_id":"semeval2015.d003.s019.t001","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s019.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":504,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003388,"input_uncached_usd":0.0000252,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.90570070798276}
{"item_id":"semeval2015.d003.s019.t003","gold_sense_keys":["only%4:02:01::"],"candidates":[{"index":1,"sense_key":"only%4:02:02::","synset_id":"merely.r.01"},{"index":2,"sense_key":"only%4:02:01::","synset_id":"entirely.r.02"},{"index":3,"sense_key":"only%4:02:00::","synset_id":"only.r.03"},{"index":4,"sense_key":"only%4:02:04::","synset_id":"only.r.04"},{"index":5,"sense_key":"only%4:02:06::","synset_id":"only.r.05"},{"index":6,"sense_key":"only%4:02:03::","synset_id":"only.r.06"},{"index":7,"sense_key":"only%4:02:05::","synset_id":"only.r.07"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"only%4:02:01::","call_ids":["semeval2015.d003.s019.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"only%4:02:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":607,"cached_input_tokens":0,"output_tokens":1296,"reasoning_output_tokens":1280},"cost":{"currency":"USD","total_usd":0.00054875,"input_uncached_usd":0.00003035,"input_cached_usd":0.0,"output_usd":0.0005184,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":7.399968999961857}
{"item_id":"semeval2015.d003.s019.t005","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"other%5:00:02:past:00","call_ids":["semeval2015.d003.s019.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"other%5:00:02:past:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.00069705,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.735922250023577}
{"item_id":"semeval2015.d003.s019.t007","gold_sense_keys":["measure%1:04:01::"],"candidates":[{"index":1,"sense_key":"measure%1:04:01::","synset_id":"measure.n.01"},{"index":2,"sense_key":"measure%1:03:00::","synset_id":"measure.n.02"},{"index":3,"sense_key":"measure%1:10:02::","synset_id":"bill.n.01"},{"index":4,"sense_key":"measure%1:04:00::","synset_id":"measurement.n.01"},{"index":5,"sense_key":"measure%1:10:03::","synset_id":"standard.n.01"},{"index":6,"sense_key":"measure%1:10:01::","synset_id":"meter.n.03"},{"index":7,"sense_key":"measure%1:10:00::","synset_id":"measure.n.07"},{"index":8,"sense_key":"measure%1:06:00::","synset_id":"measuring_stick.n.01"},{"index":9,"sense_key":"measure%1:06:01::","synset_id":"measure.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"measure%1:04:01::","call_ids":["semeval2015.d003.s019.t007__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"measure%1:04:01::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":726,"cached_input_tokens":0,"output_tokens":592,"reasoning_output_tokens":576},"cost":{"currency":"USD","total_usd":0.00027309999999999997,"input_uncached_usd":0.0000363,"input_cached_usd":0.0,"output_usd":0.00023679999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.267497499997262}
{"item_id":"semeval2015.d003.s019.t008","gold_sense_keys":["other%3:00:00::"],"candidates":[{"index":1,"sense_key":"other%3:00:00::","synset_id":"other.a.01"},{"index":2,"sense_key":"other%5:00:00:past:00","synset_id":"other.s.01"},{"index":3,"sense_key":"other%5:00:02:past:00","synset_id":"early.s.01"},{"index":4,"sense_key":"other%5:00:00:strange:00","synset_id":"other.s.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"other%3:00:00::","call_ids":["semeval2015.d003.s019.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"other%3:00:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":501,"cached_input_tokens":0,"output_tokens":1040,"reasoning_output_tokens":1024},"cost":{"currency":"USD","total_usd":0.00044104999999999996,"input_uncached_usd":0.00002505,"input_cached_usd":0.0,"output_usd":0.000416,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.186892124998849}
{"item_id":"semeval2015.d003.s019.t011","gold_sense_keys":["address%2:34:00::"],"candidates":[{"index":1,"sense_key":"address%2:32:00::","synset_id":"address.v.01"},{"index":2,"sense_key":"address%2:32:02::","synset_id":"address.v.02"},{"index":3,"sense_key":"address%2:32:03::","synset_id":"address.v.03"},{"index":4,"sense_key":"address%2:33:00::","synset_id":"address.v.04"},{"index":5,"sense_key":"address%2:34:00::","synset_id":"address.v.05"},{"index":6,"sense_key":"address%2:41:08::","synset_id":"address.v.06"},{"index":7,"sense_key":"address%2:40:00::","synset_id":"address.v.07"},{"index":8,"sense_key":"address%2:32:04::","synset_id":"cover.v.05"},{"index":9,"sense_key":"address%2:32:01::","synset_id":"address.v.09"},{"index":10,"sense_key":"address%2:30:00::","synset_id":"address.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":5,"chosen_sense_key":"address%2:34:00::","call_ids":["semeval2015.d003.s019.t011__v1__a1"],"invalid_reason":null}],"predicted_sense_index":5,"predicted_sense_key":"address%2:34:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":666,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00037249999999999995,"input_uncached_usd":0.000033299999999999996,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.137865416996647}
{"item_id":"semeval2015.d003.s019.t012","gold_sense_keys":["underlying%5:00:00:implicit:00"],"candidates":[{"index":1,"sense_key":"underlying%5:00:00:implicit:00","synset_id":"implicit_in.s.01"},{"index":2,"sense_key":"underlying%5:00:00:subjacent:00","synset_id":"underlying.s.02"},{"index":3,"sense_key":"underlying%5:00:00:basic:00","synset_id":"fundamental.s.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"underlying%5:00:00:basic:00","call_ids":["semeval2015.d003.s019.t012__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"underlying%5:00:00:basic:00","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":457,"cached_input_tokens":0,"output_tokens":272,"reasoning_output_tokens":256},"cost":{"currency":"USD","total_usd":0.00013164999999999997,"input_uncached_usd":0.00002285,"input_cached_usd":0.0,"output_usd":0.00010879999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.6229624589905143}
{"item_id":"semeval2015.d003.s019.t013","gold_sense_keys":["cause%1:10:00::","cause%1:11:00::"],"candidates":[{"index":1,"sense_key":"cause%1:11:00::","synset_id":"cause.n.01"},{"index":2,"sense_key":"cause%1:10:00::","synset_id":"cause.n.02"},{"index":3,"sense_key":"cause%1:04:01::","synset_id":"campaign.n.02"},{"index":4,"sense_key":"cause%1:03:00::","synset_id":"causal_agent.n.01"},{"index":5,"sense_key":"cause%1:04:00::","synset_id":"lawsuit.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"cause%1:03:00::","call_ids":["semeval2015.d003.s019.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"cause%1:03:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":569,"cached_input_tokens":0,"output_tokens":1424,"reasoning_output_tokens":1408},"cost":{"currency":"USD","total_usd":0.00059805,"input_uncached_usd":0.00002845,"input_cached_usd":0.0,"output_usd":0.0005696,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.210742334020324}
{"item_id":"semeval2015.d003.s020.t000","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["semeval2015.d003.s020.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":514,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.00021129999999999999,"input_uncached_usd":0.000025699999999999998,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.5477789160213433}
{"item_id":"semeval2015.d003.s021.t000","gold_sense_keys":["generally%4:02:02::"],"candidates":[{"index":1,"sense_key":"generally%4:02:02::","synset_id":"by_and_large.r.01"},{"index":2,"sense_key":"generally%4:02:01::","synset_id":"generally.r.02"},{"index":3,"sense_key":"generally%4:02:00::","synset_id":"broadly.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"generally%4:02:00::","call_ids":["semeval2015.d003.s021.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"generally%4:02:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":476,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.0005678,"input_uncached_usd":0.0000238,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.26319941697875}
{"item_id":"semeval2015.d003.s021.t001","gold_sense_keys":["well%4:02:00::"],"candidates":[{"index":1,"sense_key":"well%4:02:00::","synset_id":"well.r.01"},{"index":2,"sense_key":"well%4:02:05::","synset_id":"well.r.02"},{"index":3,"sense_key":"well%4:02:03::","synset_id":"well.r.03"},{"index":4,"sense_key":"well%4:02:15::","synset_id":"well.r.04"},{"index":5,"sense_key":"well%4:02:08::","synset_id":"well.r.05"},{"index":6,"sense_key":"well%4:02:07::","synset_id":"well.r.06"},{"index":7,"sense_key":"well%4:02:11::","synset_id":"well.r.07"},{"index":8,"sense_key":"well%4:02:14::","synset_id":"well.r.08"},{"index":9,"sense_key":"well%4:02:13::","synset_id":"well.r.09"},{"index":10,"sense_key":"well%4:02:12::","synset_id":"well.r.10"},{"index":11,"sense_key":"well%4:02:10::","synset_id":"well.r.11"},{"index":12,"sense_key":"well%4:02:09::","synset_id":"well.r.12"},{"index":13,"sense_key":"well%4:02:01::","synset_id":"well.r.13"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":13,"chosen_sense_key":"well%4:02:01::","call_ids":["semeval2015.d003.s021.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":13,"predicted_sense_key":"well%4:02:01::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":916,"cached_input_tokens":0,"output_tokens":2258,"reasoning_output_tokens":2240},"cost":{"currency":"USD","total_usd":0.000949,"input_uncached_usd":0.000045799999999999995,"input_cached_usd":0.0,"output_usd":0.0009031999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.202047541039065}
{"item_id":"semeval2015.d003.s021.t002","gold_sense_keys":["tolerate%2:39:00::"],"candidates":[{"index":1,"sense_key":"tolerate%2:31:00::","synset_id":"digest.v.03"},{"index":2,"sense_key":"tolerate%2:41:01::","synset_id":"tolerate.v.02"},{"index":3,"sense_key":"tolerate%2:39:00::","synset_id":"tolerate.v.03"},{"index":4,"sense_key":"tolerate%2:32:01::","synset_id":"allow.v.10"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"tolerate%2:39:00::","call_ids":["semeval2015.d003.s021.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"tolerate%2:39:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":547,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018734999999999997,"input_uncached_usd":0.000027349999999999997,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":2.903173957951367}
{"item_id":"semeval2015.d003.s021.t003","gold_sense_keys":["administer%2:29:00::"],"candidates":[{"index":1,"sense_key":"administer%2:41:00::","synset_id":"administer.v.01"},{"index":2,"sense_key":"administer%2:40:01::","synset_id":"administer.v.02"},{"index":3,"sense_key":"administer%2:40:00::","synset_id":"distribute.v.01"},{"index":4,"sense_key":"administer%2:29:00::","synset_id":"administer.v.04"},{"index":5,"sense_key":"administer%2:41:01::","synset_id":"administer.v.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":4,"chosen_sense_key":"administer%2:29:00::","call_ids":["semeval2015.d003.s021.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":4,"predicted_sense_key":"administer%2:29:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":551,"cached_input_tokens":0,"output_tokens":400,"reasoning_output_tokens":384},"cost":{"currency":"USD","total_usd":0.00018754999999999998,"input_uncached_usd":0.00002755,"input_cached_usd":0.0,"output_usd":0.00015999999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.921565000026021}
{"item_id":"semeval2015.d003.s021.t004","gold_sense_keys":["daily%4:02:00::"],"candidates":[{"index":1,"sense_key":"daily%4:02:00::","synset_id":"daily.r.01"},{"index":2,"sense_key":"daily%4:02:01::","synset_id":"day_by_day.r.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"daily%4:02:00::","call_ids":["semeval2015.d003.s021.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"daily%4:02:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":409,"cached_input_tokens":0,"output_tokens":912,"reasoning_output_tokens":896},"cost":{"currency":"USD","total_usd":0.00038524999999999996,"input_uncached_usd":0.00002045,"input_cached_usd":0.0,"output_usd":0.0003648,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.852113166998606}
{"item_id":"semeval2015.d003.s021.t005","gold_sense_keys":["dosage%1:06:00::"],"candidates":[{"index":1,"sense_key":"dosage%1:23:00::","synset_id":"dose.n.02"},{"index":2,"sense_key":"dosage%1:06:00::","synset_id":"dose.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dosage%1:23:00::","call_ids":["semeval2015.d003.s021.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dosage%1:23:00::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":416,"cached_input_tokens":0,"output_tokens":2193,"reasoning_output_tokens":2176},"cost":{"currency":"USD","total_usd":0.0008979999999999998,"input_uncached_usd":0.000020799999999999997,"input_cached_usd":0.0,"output_usd":0.0008772,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":13.67210983397672}
{"item_id":"semeval2015.d003.s021.t008","gold_sense_keys":["dose%1:06:00::"],"candidates":[{"index":1,"sense_key":"dose%1:06:00::","synset_id":"dose.n.01"},{"index":2,"sense_key":"dose%1:23:00::","synset_id":"dose.n.02"},{"index":3,"sense_key":"dose%1:26:00::","synset_id":"venereal_disease.n.01"},{"index":4,"sense_key":"dose%1:06:01::","synset_id":"acid.n.02"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"dose%1:06:00::","call_ids":["semeval2015.d003.s021.t008__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"dose%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":513,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028805,"input_uncached_usd":0.00002565,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.517368458036799}
{"item_id":"semeval2015.d003.s021.t009","gold_sense_keys":["extended%5:00:00:long:02"],"candidates":[{"index":1,"sense_key":"extended%5:00:00:long:02","synset_id":"drawn-out.s.01"},{"index":2,"sense_key":"extended%3:00:00::","synset_id":"extended.a.02"},{"index":3,"sense_key":"extended%5:00:00:long:01","synset_id":"elongated.s.01"},{"index":4,"sense_key":"extended%5:00:00:figurative:00","synset_id":"extended.s.03"},{"index":5,"sense_key":"extended%5:00:00:large:00","synset_id":"extensive.s.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"extended%5:00:00:long:02","call_ids":["semeval2015.d003.s021.t009__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"extended%5:00:00:long:02","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":581,"cached_input_tokens":0,"output_tokens":1360,"reasoning_output_tokens":1344},"cost":{"currency":"USD","total_usd":0.00057305,"input_uncached_usd":0.000029049999999999998,"input_cached_usd":0.0,"output_usd":0.000544,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":8.099206709011924}
{"item_id":"semeval2015.d003.s021.t013","gold_sense_keys":["duration%1:07:00::","duration%1:28:02::"],"candidates":[{"index":1,"sense_key":"duration%1:28:02::","synset_id":"duration.n.01"},{"index":2,"sense_key":"duration%1:28:00::","synset_id":"duration.n.02"},{"index":3,"sense_key":"duration%1:07:00::","synset_id":"duration.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"duration%1:28:02::","call_ids":["semeval2015.d003.s021.t013__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"duration%1:28:02::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":445,"cached_input_tokens":0,"output_tokens":720,"reasoning_output_tokens":704},"cost":{"currency":"USD","total_usd":0.00031025,"input_uncached_usd":0.00002225,"input_cached_usd":0.0,"output_usd":0.000288,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":6.330727791006211}
{"item_id":"semeval2015.d003.s021.t014","gold_sense_keys":["treatment%1:04:00::"],"candidates":[{"index":1,"sense_key":"treatment%1:04:00::","synset_id":"treatment.n.01"},{"index":2,"sense_key":"treatment%1:04:01::","synset_id":"treatment.n.02"},{"index":3,"sense_key":"treatment%1:07:00::","synset_id":"treatment.n.03"},{"index":4,"sense_key":"treatment%1:10:00::","synset_id":"discussion.n.01"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"treatment%1:04:00::","call_ids":["semeval2015.d003.s021.t014__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"treatment%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":518,"cached_input_tokens":0,"output_tokens":656,"reasoning_output_tokens":640},"cost":{"currency":"USD","total_usd":0.00028829999999999996,"input_uncached_usd":0.0000259,"input_cached_usd":0.0,"output_usd":0.0002624,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.377224707975984}
{"item_id":"semeval2015.d003.s022.t000","gold_sense_keys":["precaution%1:04:00::"],"candidates":[{"index":1,"sense_key":"precaution%1:04:00::","synset_id":"precaution.n.01"},{"index":2,"sense_key":"precaution%1:07:00::","synset_id":"precaution.n.02"},{"index":3,"sense_key":"precaution%1:09:00::","synset_id":"caution.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"precaution%1:04:00::","call_ids":["semeval2015.d003.s022.t000__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"precaution%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":464,"reasoning_output_tokens":448},"cost":{"currency":"USD","total_usd":0.000208,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.00018559999999999998,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":3.0962465000338852}
{"item_id":"semeval2015.d003.s022.t001","gold_sense_keys":["person%1:03:00::"],"candidates":[{"index":1,"sense_key":"person%1:03:00::","synset_id":"person.n.01"},{"index":2,"sense_key":"person%1:08:00::","synset_id":"person.n.02"},{"index":3,"sense_key":"person%1:10:00::","synset_id":"person.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"person%1:03:00::","call_ids":["semeval2015.d003.s022.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"person%1:03:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":448,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036159999999999995,"input_uncached_usd":0.0000224,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.192454665957484}
{"item_id":"semeval2015.d003.s022.t002","gold_sense_keys":["give%2:40:13::"],"candidates":[{"index":1,"sense_key":"give%2:40:03::","synset_id":"give.v.01"},{"index":2,"sense_key":"give%2:40:05::","synset_id":"yield.v.01"},{"index":3,"sense_key":"give%2:40:00::","synset_id":"give.v.03"},{"index":4,"sense_key":"give%2:40:07::","synset_id":"give.v.04"},{"index":5,"sense_key":"give%2:32:01::","synset_id":"give.v.05"},{"index":6,"sense_key":"give%2:36:02::","synset_id":"hold.v.03"},{"index":7,"sense_key":"give%2:32:00::","synset_id":"give.v.07"},{"index":8,"sense_key":"give%2:40:01::","synset_id":"give.v.08"},{"index":9,"sense_key":"give%2:36:03::","synset_id":"give.v.09"},{"index":10,"sense_key":"give%2:31:02::","synset_id":"give.v.10"},{"index":11,"sense_key":"give%2:36:00::","synset_id":"render.v.04"},{"index":12,"sense_key":"give%2:40:09::","synset_id":"impart.v.01"},{"index":13,"sense_key":"give%2:36:01::","synset_id":"establish.v.05"},{"index":14,"sense_key":"give%2:40:02::","synset_id":"give.v.14"},{"index":15,"sense_key":"give%2:41:00::","synset_id":"give.v.15"},{"index":16,"sense_key":"give%2:40:10::","synset_id":"sacrifice.v.01"},{"index":17,"sense_key":"give%2:40:06::","synset_id":"pass.v.05"},{"index":18,"sense_key":"give%2:32:03::","synset_id":"give.v.18"},{"index":19,"sense_key":"give%2:40:13::","synset_id":"give.v.19"},{"index":20,"sense_key":"give%2:40:14::","synset_id":"give.v.20"},{"index":21,"sense_key":"give%2:32:02::","synset_id":"give.v.21"},{"index":22,"sense_key":"give%2:40:11::","synset_id":"grant.v.05"},{"index":23,"sense_key":"give%2:38:00::","synset_id":"move_over.v.01"},{"index":24,"sense_key":"give%2:34:00::","synset_id":"feed.v.02"},{"index":25,"sense_key":"give%2:40:08::","synset_id":"contribute.v.02"},{"index":26,"sense_key":"give%2:38:13::","synset_id":"collapse.v.01"},{"index":27,"sense_key":"give%2:31:00::","synset_id":"give.v.27"},{"index":28,"sense_key":"give%2:41:10::","synset_id":"give.v.28"},{"index":29,"sense_key":"give%2:40:12::","synset_id":"give.v.29"},{"index":30,"sense_key":"give%2:40:15::","synset_id":"afford.v.04"},{"index":31,"sense_key":"give%2:36:13::","synset_id":"give.v.31"},{"index":32,"sense_key":"give%2:36:12::","synset_id":"give.v.32"},{"index":33,"sense_key":"give%2:35:00::","synset_id":"give.v.33"},{"index":34,"sense_key":"give%2:34:10::","synset_id":"give.v.34"},{"index":35,"sense_key":"give%2:32:11::","synset_id":"give.v.35"},{"index":36,"sense_key":"give%2:32:08::","synset_id":"give.v.36"},{"index":37,"sense_key":"give%2:32:15::","synset_id":"give.v.37"},{"index":38,"sense_key":"give%2:32:14::","synset_id":"give.v.38"},{"index":39,"sense_key":"give%2:32:09::","synset_id":"give.v.39"},{"index":40,"sense_key":"give%2:32:07::","synset_id":"give.v.40"},{"index":41,"sense_key":"give%2:32:10::","synset_id":"give.v.41"},{"index":42,"sense_key":"give%2:30:13::","synset_id":"give.v.42"},{"index":43,"sense_key":"give%2:29:11::","synset_id":"give.v.43"},{"index":44,"sense_key":"give%2:29:10::","synset_id":"give.v.44"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":19,"chosen_sense_key":"give%2:40:13::","call_ids":["semeval2015.d003.s022.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":19,"predicted_sense_key":"give%2:40:13::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":2058,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0004165,"input_uncached_usd":0.0001029,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.361312666966114}
{"item_id":"semeval2015.d003.s022.t003","gold_sense_keys":["medicine%1:06:00::"],"candidates":[{"index":1,"sense_key":"medicine%1:09:00::","synset_id":"medicine.n.01"},{"index":2,"sense_key":"medicine%1:06:00::","synset_id":"medicine.n.02"},{"index":3,"sense_key":"medicine%1:04:00::","synset_id":"medicine.n.03"},{"index":4,"sense_key":"medicine%1:04:02::","synset_id":"music.n.05"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"medicine%1:06:00::","call_ids":["semeval2015.d003.s022.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"medicine%1:06:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":481,"cached_input_tokens":0,"output_tokens":848,"reasoning_output_tokens":832},"cost":{"currency":"USD","total_usd":0.00036324999999999997,"input_uncached_usd":0.00002405,"input_cached_usd":0.0,"output_usd":0.0003392,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":5.019324166001752}
{"item_id":"semeval2015.d003.s022.t005","gold_sense_keys":["contact%1:04:00::","contact%1:26:00::"],"candidates":[{"index":1,"sense_key":"contact%1:04:02::","synset_id":"contact.n.01"},{"index":2,"sense_key":"contact%1:04:00::","synset_id":"contact.n.02"},{"index":3,"sense_key":"contact%1:26:00::","synset_id":"contact.n.03"},{"index":4,"sense_key":"contact%1:11:00::","synset_id":"contact.n.04"},{"index":5,"sense_key":"contact%1:18:00::","synset_id":"contact.n.05"},{"index":6,"sense_key":"contact%1:10:00::","synset_id":"liaison.n.02"},{"index":7,"sense_key":"contact%1:06:00::","synset_id":"contact.n.07"},{"index":8,"sense_key":"contact%1:10:01::","synset_id":"contact.n.08"},{"index":9,"sense_key":"contact%1:06:01::","synset_id":"contact.n.09"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"contact%1:04:00::","call_ids":["semeval2015.d003.s022.t005__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"contact%1:04:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":702,"cached_input_tokens":0,"output_tokens":1872,"reasoning_output_tokens":1856},"cost":{"currency":"USD","total_usd":0.0007839,"input_uncached_usd":0.0000351,"input_cached_usd":0.0,"output_usd":0.0007488,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":10.993721959006507}
{"item_id":"semeval2015.d003.s023.t001","gold_sense_keys":["represent%2:42:00::"],"candidates":[{"index":1,"sense_key":"represent%2:42:02::","synset_id":"represent.v.01"},{"index":2,"sense_key":"represent%2:32:02::","synset_id":"typify.v.02"},{"index":3,"sense_key":"represent%2:41:02::","synset_id":"represent.v.03"},{"index":4,"sense_key":"represent%2:41:00::","synset_id":"represent.v.04"},{"index":5,"sense_key":"represent%2:32:00::","synset_id":"represent.v.05"},{"index":6,"sense_key":"represent%2:42:01::","synset_id":"exemplify.v.01"},{"index":7,"sense_key":"represent%2:42:00::","synset_id":"constitute.v.01"},{"index":8,"sense_key":"represent%2:41:06::","synset_id":"defend.v.06"},{"index":9,"sense_key":"represent%2:36:01::","synset_id":"represent.v.09"},{"index":10,"sense_key":"represent%2:36:02::","synset_id":"act.v.03"},{"index":11,"sense_key":"represent%2:36:11::","synset_id":"stage.v.01"},{"index":12,"sense_key":"represent%2:32:12::","synset_id":"represent.v.12"},{"index":13,"sense_key":"represent%2:32:10::","synset_id":"represent.v.13"},{"index":14,"sense_key":"represent%2:32:11::","synset_id":"present.v.02"},{"index":15,"sense_key":"represent%2:30:06::","synset_id":"map.v.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":12,"chosen_sense_key":"represent%2:32:12::","call_ids":["semeval2015.d003.s023.t001__v1__a1"],"invalid_reason":null}],"predicted_sense_index":12,"predicted_sense_key":"represent%2:32:12::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":953,"cached_input_tokens":0,"output_tokens":2832,"reasoning_output_tokens":2816},"cost":{"currency":"USD","total_usd":0.00118045,"input_uncached_usd":0.00004765,"input_cached_usd":0.0,"output_usd":0.0011328,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.586601334041916}
{"item_id":"semeval2015.d003.s023.t002","gold_sense_keys":["particular%5:00:00:uncommon:00","particular%5:00:02:specific:00"],"candidates":[{"index":1,"sense_key":"particular%5:00:00:specific:00","synset_id":"particular.s.01"},{"index":2,"sense_key":"particular%5:00:02:specific:00","synset_id":"particular.s.02"},{"index":3,"sense_key":"particular%5:00:00:uncommon:00","synset_id":"especial.s.01"},{"index":4,"sense_key":"particular%5:00:00:primary:00","synset_id":"particular.s.04"},{"index":5,"sense_key":"particular%5:00:00:fastidious:00","synset_id":"finical.s.01"},{"index":6,"sense_key":"particular%5:00:00:careful:00","synset_id":"particular.s.06"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":2,"chosen_sense_key":"particular%5:00:02:specific:00","call_ids":["semeval2015.d003.s023.t002__v1__a1"],"invalid_reason":null}],"predicted_sense_index":2,"predicted_sense_key":"particular%5:00:02:specific:00","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":573,"cached_input_tokens":0,"output_tokens":1680,"reasoning_output_tokens":1664},"cost":{"currency":"USD","total_usd":0.0007006499999999999,"input_uncached_usd":0.00002865,"input_cached_usd":0.0,"output_usd":0.000672,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":14.01261129201157}
{"item_id":"semeval2015.d003.s023.t003","gold_sense_keys":["risk%1:26:00::"],"candidates":[{"index":1,"sense_key":"risk%1:26:00::","synset_id":"hazard.n.01"},{"index":2,"sense_key":"risk%1:04:00::","synset_id":"risk.n.02"},{"index":3,"sense_key":"risk%1:07:02::","synset_id":"risk.n.03"},{"index":4,"sense_key":"risk%1:07:01::","synset_id":"risk.n.04"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":1,"chosen_sense_key":"risk%1:26:00::","call_ids":["semeval2015.d003.s023.t003__v1__a1"],"invalid_reason":null}],"predicted_sense_index":1,"predicted_sense_key":"risk%1:26:00::","is_correct":true,"status":"success","was_monosemous":false,"usage":{"input_tokens":452,"cached_input_tokens":0,"output_tokens":784,"reasoning_output_tokens":768},"cost":{"currency":"USD","total_usd":0.0003362,"input_uncached_usd":0.0000226,"input_cached_usd":0.0,"output_usd":0.0003136,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":4.466722542012576}
{"item_id":"semeval2015.d003.s023.t004","gold_sense_keys":["people%1:14:00::","people%1:14:03::"],"candidates":[{"index":1,"sense_key":"people%1:14:00::","synset_id":"people.n.01"},{"index":2,"sense_key":"people%1:14:01::","synset_id":"citizenry.n.01"},{"index":3,"sense_key":"people%1:14:02::","synset_id":"people.n.03"},{"index":4,"sense_key":"people%1:14:03::","synset_id":"multitude.n.03"}],"votes":[{"vote_index":1,"status":"success","chosen_sense_index":3,"chosen_sense_key":"people%1:14:02::","call_ids":["semeval2015.d003.s023.t004__v1__a1"],"invalid_reason":null}],"predicted_sense_index":3,"predicted_sense_key":"people%1:14:02::","is_correct":false,"status":"success","was_monosemous":false,"usage":{"input_tokens":450,"cached_input_tokens":0,"output_tokens":2640,"reasoning_output_tokens":2624},"cost":{"currency":"USD","total_usd":0.0010784999999999998,"input_uncached_usd":0.000022499999999999998,"input_cached_usd":0.0,"output_usd":0.0010559999999999999,"input_uncached_unit_price_usd":5e-8,"input_cached_unit_price_usd":5e-9,"output_unit_price_usd":4e-7,"source":"litellm_estimate"},"latency_seconds":15.65237949998118}